From 5f300963a07ad8f8635edd24312463ed3b6638e1 Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Wed, 20 Dec 2017 22:32:30 -0800
Subject: [PATCH] Updated UI for Named Entities Tab under Page Options

---
 src/controllers/components/CrawlComponent.php |  88 ++++++++++++++--
 src/locale/en_US/configure.ini                |   4 +
 src/models/EntityModel.php                    | 137 +++++++++++++++++++++++++
 src/views/elements/PageoptionsElement.php     | 141 ++++++++++++++++++++++++--
 4 files changed, 352 insertions(+), 18 deletions(-)
 create mode 100644 src/models/EntityModel.php

diff --git a/src/controllers/components/CrawlComponent.php b/src/controllers/components/CrawlComponent.php
index 698b7ed6..773b0cd8 100644
--- a/src/controllers/components/CrawlComponent.php
+++ b/src/controllers/components/CrawlComponent.php
@@ -1023,6 +1023,7 @@ class CrawlComponent extends Component implements CrawlConstants
         $profile_model = $parent->model("profile");
         $data["ELEMENT"] = "pageoptions";
         $data['SCRIPT'] = "";
+
         $machine_urls = $parent->model("machine")->getQueueServerUrls();
         $num_machines = count($machine_urls);
         if ($num_machines <  1 || ($num_machines ==  1 &&
@@ -1319,7 +1320,8 @@ class CrawlComponent extends Component implements CrawlConstants
         } else {
             $data['page_rules'] = "";
         }
-        $allowed_options = ['crawl_time', 'search_time', 'test_options'];
+        $allowed_options = ['crawl_time', 'search_time', 'named_entity',
+            'test_options'];
         if (isset($_REQUEST['option_type']) &&
             in_array($_REQUEST['option_type'], $allowed_options)) {
             $data['option_type'] = $_REQUEST['option_type'];
@@ -1329,25 +1331,35 @@ class CrawlComponent extends Component implements CrawlConstants
         if ($data['option_type'] == 'crawl_time') {
             $data['crawl_time_active'] = "active";
             $data['search_time_active'] = "";
+            $data['named_entity_active'] = "";
             $data['test_options_active'] = "";
             $data['SCRIPT'] .= "\nswitchTab('crawltimetab',".
-                "'searchtimetab', 'testoptionstab')\n";
+                "'searchtimetab', 'namedentitytab', 'testoptionstab')\n";
         } else if ($data['option_type'] == 'search_time') {
             $data['search_time_active'] = "active";
             $data['crawl_time_active'] = "";
+            $data['named_entity_active'] = "";
             $data['test_options_active'] = "";
             $data['SCRIPT'] .= "\nswitchTab('searchtimetab',".
-                "'crawltimetab', 'testoptionstab')\n";
+                "'crawltimetab', 'namedentitytab', 'testoptionstab')\n";
+        } else if ($data['option_type'] == 'named_entity') {
+            $data['search_time_active'] = "";
+            $data['crawl_time_active'] = "";
+            $data['named_entity_active'] = "active";
+            $data['test_options_active'] = "";
+            $data['SCRIPT'] .= "\nswitchTab('namedentitytab'," .
+                "'crawltimetab', 'searchtimetab', 'testoptionstab');\n";
         } else {
             $data['search_time_active'] = "";
             $data['crawl_time_active'] = "";
+            $data['named_entity_active'] = "";
             $data['test_options_active'] = "active";
             $data['SCRIPT'] .= "\nswitchTab('testoptionstab',".
-                "'crawltimetab', 'searchtimetab');\n";
+                "'crawltimetab', 'searchtimetab', 'namedentitytab');\n";
         }
-
         $crawl_model->setSeedInfo($seed_info);
-        if ($change == true && $data['option_type'] != 'test_options') {
+        if ($change == true && $data['option_type'] != 'test_options' &&
+            $data['option_type'] != 'named_entity') {
             return $parent->redirectWithMessage(
                 tl('crawl_component_page_options_updated'),
                 ["option_type"]);
@@ -1550,10 +1562,38 @@ class CrawlComponent extends Component implements CrawlConstants
             $data['PROCESS_TIMES']['TOTAL'] = L\changeInMicrotime($start_time);
             $data['PROCESS_TIMES'] = array_merge($data['PROCESS_TIMES'],
                 $word_lists['TIMES']);
-        }
+        } else if ($data['option_type'] == 'named_entity') {
+            $data= array_merge($data, CrawlComponent::addNamedEntites($data));
+        }
+        $languages = $parent->model("locale")->getLocaleList();
+        foreach ($languages as $language) {
+            $data['LOCALES'][$language['LOCALE_TAG']] =
+                $language['LOCALE_NAME'];
+        }
+        $entity_model = $parent->model("entity");
+        $locale_model = $parent->model("locale");
+        $locale_tag = L\getLocaleTag();
+        $data['CURRENT_LOCALE_TAG'] =
+        !empty($_REQUEST['LOCALE_TAG']) ? $_REQUEST['LOCALE_TAG'] : $locale_tag;
+        //$locale_tag = $data['CURRENT_LOCALE_TAG'];
+        $locale_tag = $data['CURRENT_LOCALE_TAG'];
+        $data['CURRENT_LOCALE_NAME'] =
+                        $locale_model->getLocaleName($locale_tag);
+        $data['ENTITIES'] = $entity_model->viewall($locale_tag);
+        $data['NUM_ENTITIES_SHOW'] = 100;
+        $data['TOTAL_ENTITIES'] = count($data['ENTITIES']);
+        $data['LIMIT'] = (isset($_REQUEST['limit'])) ?
+            min($parent->clean($_REQUEST['limit'], 'int'),
+            $data['TOTAL_ENTITIES']) : 0;
+        $data['ENTITIES'] = array_slice($data['ENTITIES'],
+            $data['LIMIT'], $data['NUM_ENTITIES_SHOW']);
+        $data['upload_options'] = [tl('pageoptions_element_actions'),
+         tl('pageoptions_element_add_entity'),
+         tl('pageoptions_element_file_upload')];
+        $data['entity_default'] = [tl('pageoptions_element_actions')];
         return $data;
     }
-    /**
+    /**$data['SCRIPT'] .= 'entities = ' . $data['ENTITIES']
      * Handles admin request related to the Scrapers activity
      *
      * This activity allows a user to specify the configuration for the
@@ -2091,4 +2131,36 @@ class CrawlComponent extends Component implements CrawlConstants
             "switchSourceType()";
         return $data;
     }
+
+    public function addNamedEntites($data)
+    {
+        $parent = $this->parent;
+        $entity_model = $parent->model("entity");
+        $locale_model = $parent->model("locale");
+        if (isset($_REQUEST['ENTITY_NAME'])) {
+            $entity = $_REQUEST['ENTITY_NAME'];
+            $locale = $_REQUEST['LOCALE_TAG'];
+            $entity_model->addentity($entity, $locale);
+        } else if (isset($_FILES['ENTITY_FILE'])) {
+            $file_name = $_FILES['ENTITY_FILE']['tmp_name'];
+            if (!empty($file_name)) {
+                $locale = $_REQUEST['LOCALE_TAG'];
+                $entity_model->addEntities($file_name, $locale);
+            }
+        } else {
+            if (!empty($_REQUEST['arg'])) {
+                switch ($_REQUEST['arg']) {
+                    case 'deleteentities':
+                        if (isset($_REQUEST['value']) &&
+                            isset($_REQUEST['LOCALE_TAG'])) {
+                            $entity = $_REQUEST['value'];
+                            $locale = $_REQUEST['LOCALE_TAG'];
+                            $entity_model->deleteentity($entity, $locale);
+                        }
+                        break;
+                }
+            }
+        }
+        return $data;
+    }
 }
diff --git a/src/locale/en_US/configure.ini b/src/locale/en_US/configure.ini
index cf05464e..6f4e31c9 100644
--- a/src/locale/en_US/configure.ini
+++ b/src/locale/en_US/configure.ini
@@ -1146,6 +1146,7 @@ moreoptions_element_tools = "Tools"
 ; PageoptionsElement.php
 pageoptions_element_crawl_time = "Crawl Time"
 pageoptions_element_search_time = "Search Time"
+pageoptions_element_named_entity = "Named Entity"
 pageoptions_element_test_options = "Test Options"
 pageoptions_element_load_options = "Get Page Options From:"
 pageoptions_element_page_range = "Byte Range to Download (0 - Value):"
@@ -1200,6 +1201,9 @@ pageoptions_element_qa_time = "Question Answer Extraction:"
 pageoptions_element_total_time = "Total Time:"
 pageoptions_element_time_seconds = "%s seconds"
 pageoptions_element_plugin_back = "Back"
+pageoptions_element_actions = "Actions"
+pageoptions_element_add_entity = "Add Entity"
+pageoptions_element_file_upload = "Upload File"
 ;
 ; QuerystatsElement.php
 querystats_element_back_to_manage = "Back"
diff --git a/src/models/EntityModel.php b/src/models/EntityModel.php
new file mode 100644
index 00000000..9d9e2ccc
--- /dev/null
+++ b/src/models/EntityModel.php
@@ -0,0 +1,137 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2017  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2017
+ * @filesource
+ */
+namespace seekquarry\yioop\models;
+
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+
+/**
+ * This class is used to handle
+ * database statements related adding entities
+ *
+ * @author Salil Shenoy
+ */
+class EntityModel extends Model
+{
+    /**
+     * Add an entity to the database using provided string
+     *
+     * @param string $entity the enityt to be added
+     * @param string $locale the locale to which the entity belongs
+     */
+    public function addEntity($entity, $locale)
+    {
+        $db = $this->db;
+        $sql = "INSERT INTO LEXICON(TERM, PART_OF_SPEECH, LOCALE) VALUES
+         (?,?,?)";
+        $result = $db->execute($sql, [$entity, "EN", $locale]);
+    }
+    /**
+     * Bulk upload entities to the database from a file for 
+     * the given locale
+     *
+     * @param file $file_name containing entities uploaded to Yioop
+     * @param string $locale locale to which the entities belong
+     */
+    public function addEntities($file_name, $locale)
+    {
+        $db = $this->db;
+        $lines = file($file_name);
+        $sql = "INSERT INTO LEXICON(TERM, PART_OF_SPEECH, LOCALE) VALUES
+         (?,?,?)";
+        foreach ($lines as $line) {
+            $result = $db->execute($sql, [$line, "EN", $locale]);     
+        }
+    }
+    /**
+     * Bulk edit entities already present in the database from the View 
+     * Entities page the given locale.
+     *
+     * @param array $orignal_values of entities present in Yioop
+     * @param array $updated_values captured when the user clicks Save button
+     * @param string $locale locale to which the entities belong
+     */
+    public function bulkEditEntities($orignal_values, $updated_values, $locale)
+    {
+        $db = $this->db;
+        $sql = "UPDATE LEXICON SET TERM = ? WHERE TERM = ? AND LOCALE= ?";
+        /* The following code gets only the entites which are updated 
+         *  For example,
+         *  $orignal_values = [0 => 'New Jersey', 1 => 'California']
+         *  $update_values = [ 0 => 'New York', 1 => 'California']
+         *  $old_values = [0 => 'New Jersey']
+         *  $new_values = [0 => 'New York']
+         */
+        $old_values = array_diff($orignal_values, $updated_values);
+        $new_values = array_diff($updated_values, $orignal_values);
+        $count = count($new_values);
+        if ($count > 0) {
+            foreach ($new_values as $key => $value) {
+                $new_value = $new_values[$key];
+                $old_value = $old_values[$key];
+                $db->execute($sql, [$new_value, $old_value, $locale]);
+                $count--;
+                if ($count == 0)
+                    break;
+            }
+        }
+    }
+    /**
+     * Shows all the entities foe that locale, you can then edit or delete
+     * entites from this page
+     *
+     * @param string $locale locale to which the entities belong
+     */
+    public function viewall($locale) 
+    {
+        $entities = [];
+        $db = $this->db;
+        $sql = "SELECT term FROM LEXICON WHERE LOCALE = ? AND
+            PART_OF_SPEECH = 'EN'";
+        $result = $db->execute($sql, [$locale]);
+        while ($row = $db->fetchArray($result)){
+            $entities[] = $row['TERM'];
+        }
+        return $entities;
+    }
+    /**
+     * Delete the entity for the locale
+     *
+     * @param string $entity entity to delete
+     * @param string $locale locale to which the entities belong
+     */
+    public function deleteentity($entity, $locale)
+    {
+        $db = $this->db;
+        $sql = "DELETE FROM LEXICON WHERE TERM = ? AND LOCALE = ?";
+        $db->execute($sql, [$entity, $locale]);
+    }
+}
diff --git a/src/views/elements/PageoptionsElement.php b/src/views/elements/PageoptionsElement.php
index 3f031df9..7be89e92 100644
--- a/src/views/elements/PageoptionsElement.php
+++ b/src/views/elements/PageoptionsElement.php
@@ -31,6 +31,7 @@
 namespace seekquarry\yioop\views\elements;
 
 use seekquarry\yioop\configs as C;
+use seekquarry\yioop as B;
 
 /**
  * This element is used to render the Page Options admin activity
@@ -51,20 +52,30 @@ class PageOptionsElement extends Element
     {
     ?>
         <div class="current-activity">
-        <form id="pageoptionsForm" method="post" action='?'>
+        <form id="pageoptionsForm" enctype="multipart/form-data"
+         method="post" action='?'>
         <ul class='tab-menu-list'>
         <li><a href="javascript:
-                switchTab('crawltimetab', 'searchtimetab', 'testoptionstab');"
+                switchTab('crawltimetab', 'searchtimetab', 'namedentitytab',
+                    'testoptionstab');"
             id='crawltimetabitem'
             class="<?= $data['crawl_time_active'] ?>"><?=
             tl('pageoptions_element_crawl_time')?></a></li>
         <li><a href="javascript:
-                switchTab('searchtimetab', 'crawltimetab', 'testoptionstab');"
+                switchTab('searchtimetab', 'crawltimetab', 'namedentitytab',
+                    'testoptionstab');"
             id='searchtimetabitem'
             class="<?= $data['search_time_active'] ?>"><?=
             tl('pageoptions_element_search_time')?></a></li>
         <li><a href="javascript:
-                switchTab('testoptionstab', 'crawltimetab', 'searchtimetab');"
+                switchTab('namedentitytab', 'crawltimetab', 'searchtimetab',
+                    'testoptionstab');"
+            id='namedentitytabitem'
+            class="<? = $data['named_entity_active'] ?>"><?=
+            tl('pageoptions_element_named_entity') ?></a></li>
+        <li><a href="javascript:
+                switchTab('testoptionstab', 'crawltimetab', 'searchtimetab',
+                'namedentitytab');"
             id='testoptionstabitem'
             class="<?= $data['test_options_active'] ?>"><?=
             tl('pageoptions_element_test_options') ?></a></li>
@@ -355,6 +366,61 @@ class PageOptionsElement extends Element
                 value="<?= $data['SERVER_ALPHA'] ?>" /></td></tr>
         </table>
         </div>
+        <div id='namedentitytab'>
+        <?php
+        $locale_list = [];
+        foreach ($data['LOCALES'] as $locale_tag => $locale_name) {
+            $locale_list[] = $locale_tag;
+        }
+        ?>
+        <input type="hidden" name="arg" value="save-entities" />
+        <h2>Entities: <?php echo $data['CURRENT_LOCALE_NAME'] ?></h2>
+        <table id="entity-table" class="entity-table">
+        <tr>
+        <th>Locale: <?=  $this->view->helper("options")->render("LOCALE_TAG",
+            "LOCALE_TAG", array_combine($locale_list, $locale_list),
+            $data['CURRENT_LOCALE_TAG'], true); ?></th>
+        <th>
+        <?php
+            $this->view->helper("options")->render("ADD_ENTITY_OPTION",
+            "ADD_ENTITY_OPTION", $data['upload_options'],
+             $data['entity_default'], "uploadOptions();");
+        ?>
+        </th>
+        </tr>
+        <?php
+        $admin_url = htmlentities(B\controllerUrl('admin', true));
+        $base_url = $admin_url ."a=pageOptions&amp;".
+            C\CSRF_TOKEN . "=" . $data[C\CSRF_TOKEN] . "&amp;selectlocale=" .
+            $data['CURRENT_LOCALE_TAG'];
+        $i = 0;
+        $delete_url = $base_url .
+        "&amp;arg=deleteentities&option_type=named_entity";
+        foreach ($data['ENTITIES'] as $entity) {
+            e("<tr><td style='display:none;'><input type='hidden' ".
+             "value='$entity' id=$i name=OLD_VALUE[$i]");
+            e("<tr><td><input type='text' ". "value='$entity'
+                id=$i name=EN[$i]/></td>");
+            $i++;
+            e("<td><a name=$entity href='$base_url"
+            ."&amp;arg=deleteentities&amp;option_type=named_entity&amp;
+            value=$entity"."'");?>
+            onclick='javascript:return confirm("<?php
+            e(tl('editentities_confirm_delete')); ?>");'><?php
+            e(tl("editentities_element_delete")."</a></td></tr>");
+        }
+        ?>
+        </table>
+        <?php
+            $this->view->helper("pagination")->render($base_url,
+                $data['LIMIT'], $data['NUM_ENTITIES_SHOW'],
+                $data['TOTAL_ENTITIES']);
+        ?>
+        <div id="save-entity-btn-space" class="center slight-pad">
+        <button type="submit" class="button-box" id="named-entity-btn">
+        <?php e(tl('pageoptions_element_save_options'));?></button></div>
+        </div>
+        </div>
         <div id='testoptionstab'>
         <h2><?= tl('pageoptions_element_test_page') ?>
         <?= $this->view->helper("helpbutton")->render(
@@ -373,8 +439,8 @@ class PageOptionsElement extends Element
 
         </div>
 
-        <div class="center slight-pad"><button class="button-box"
-            id="page-button"
+        <div id="page-btn-space" class="center slight-pad">
+            <button class="button-box" id="page-button"
             type="submit"><?php if ($data['test_options_active'] == "") {
                 e(tl('pageoptions_element_save_options'));
             } else {
@@ -459,11 +525,15 @@ class PageOptionsElement extends Element
         }
         ?>
         <script type="text/javascript">
-        function switchTab(newtab, oldtab, oldtab2)
+        function switchTab(newtab, oldtab, oldtab2, oldtab3)
         {
+            var div1 = document.getElementById('save-entity-btn-space');
+            var div2 = document.getElementById('page-btn-space');
+
             setDisplay(newtab, true);
             setDisplay(oldtab, false);
             setDisplay(oldtab2, false);
+            setDisplay(oldtab3, false);
             ntab = elt(newtab + "item");
             if (ntab) {
                 ntab.className = 'active';
@@ -476,11 +546,37 @@ class PageOptionsElement extends Element
             if (otab2) {
                 otab2.className = '';
             }
+            otab3 = elt(oldtab3 + "item");
+            if (otab3) {
+                otab3.className = '';
+            }
             ctype = elt('option-type');
             if (ctype) {
-                ctype.value = (newtab == 'crawltimetab')
-                    ? 'crawl_time' : ((newtab == 'searchtimetab') ?
-                    'search_time' : 'test_options' );
+                switch (newtab) {
+                    case 'crawltimetab':
+                        ctype.value = 'crawl_time';
+                        div1.style.display = "none";
+                        div2.style.display = "block";
+                        break;
+
+                    case 'searchtimetab':
+                        ctype.value = 'search_time';
+                        div1.style.display = "none";
+                        div2.style.display = "block";
+                        break;
+
+                    case 'namedentitytab':
+                        ctype.value = 'named_entity';
+                        div1.style.display = "block";
+                        div2.style.display = "none";
+                        break;
+
+                    default:
+                        ctype.value = 'test_options';
+                        div1.style.display = "none";
+                        div2.style.display = "block";
+                }
+
                 if (ctype.value == 'test_options') {
                     elt('page-button').innerHTML =
                         '<?= tl('pageoptions_element_run_tests') ?>';
@@ -492,6 +588,31 @@ class PageOptionsElement extends Element
                 }
             }
         }
+
+        function uploadOptions() {
+            var form = document.getElementById("pageoptionsForm");
+            var options = document.getElementById("ADD_ENTITY_OPTION");
+            var index = options.selectedIndex;
+            var text =  options[index].text;
+            if (text == '<?= tl('pageoptions_element_add_entity') ?>') {
+                var entity_name = document.createElement('input');
+                entity_name.setAttribute('name', 'ENTITY_NAME');
+                entity_name.setAttribute('type', 'text');
+                var entity = prompt("Please enter the entity:", "");
+                if (entity != null || entity != "") {
+                    entity_name.value = entity;
+                    form.appendChild(entity_name);
+                    entity_name.style.visibility = "hidden";
+                }
+            } else if (text == '<?= tl('pageoptions_element_file_upload') ?>') {
+                var fileSelector = document.createElement('input');
+                fileSelector.setAttribute('name', 'ENTITY_FILE');
+                fileSelector.setAttribute('type', 'file');
+                fileSelector.click();
+                form.appendChild(fileSelector);
+                fileSelector.style.visibility = "hidden";
+            }
+        }
         </script>
     <?php
     }
-- 
2.15.0.windows.1

