From 5ed4a49307dacca5286f38e26a969273ea8af372 Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Tue, 17 Oct 2017 18:02:33 -0700
Subject: [PATCH 1/2] Hindi Triplet Extractor, Lexicon in Database and removing
 unwanted print from Summarizer

---
 src/configs/Config.php                         |   2 +-
 src/configs/Createdb.php                       |  43 ++
 src/library/VersionFunctions.php               |  10 +-
 src/library/summarizers/CentroidSummarizer.php |   1 -
 src/locale/en_US/resources/Tokenizer.php       |   2 +-
 src/locale/hi/resources/Tokenizer.php          | 836 ++++++++++++++++++++-----
 src/locale/hi/resources/lexicon.txt.gz         | Bin 200022 -> 1791332 bytes
 7 files changed, 732 insertions(+), 162 deletions(-)

diff --git a/src/configs/Config.php b/src/configs/Config.php
index 31c8afe9..0a010845 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -112,7 +112,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade function
  * @var int
  */
-nsdefine('YIOOP_VERSION', 54);
+nsdefine('YIOOP_VERSION', 55);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index 6765d095..3430501f 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -248,6 +248,49 @@ foreach ($locales as $locale) {
     $locale_index[$locale[0]] = $i;
     $i++;
 }
+//Insert the lexicon for all locales in the database
+foreach ($locales as $locale) {
+    $folderName = $locale[0];
+    if (strstr($locale[0], "-")) {
+        $localeName = explode("-", $locale[0]);
+        $folderName = $localeName[0] . "_" . $localeName[1];
+    }
+    $lexicon_file = C\LOCALE_DIR . "/" . $folderName .
+        "/resources/lexicon.txt.gz";
+    if (file_exists($lexicon_file))
+    {
+        $lines = gzfile($lexicon_file);
+        $insert_values = "";
+        $count = 0;
+        foreach ($lines as $line) {
+            $line = trim($line, " ");
+            $line = explode(" ", $line);
+            if (empty($line[0]) || empty($line[1]))
+                continue;
+            $insert_values .= '(\'' . trim($line[0]) . '\',\'' . $locale[0] .
+                '\',\'' . trim($line[1]) . '\'),';
+            $count++;
+            if ($count >= 10000) {
+                $insert_values = rtrim($insert_values, ',');
+                $query = 'INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                    {$insert_values}';
+                $db->exec($query);
+                $insert_values = "";
+                $count = 0;
+                if ($db->affectedRows() == 0) {
+                    continue;
+                }
+            }
+        }
+        if ($count > 0)
+        {
+            $insert_values = rtrim($insert_values, ',');
+            $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                {$insert_values}";
+            $db->exec($query);
+        }
+    }
+}
 $activities = [
     "manageAccount" => ['db_activity_manage_account',
         [
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index a307d5d8..6131a945 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1444,7 +1444,6 @@ function upgradeDatabaseVersion51(&$db){
  */
 function upgradeDatabaseVersion52(&$db)
 {
-
     $db->execute("CREATE TABLE ITEM_IMPRESSION_STAT(ITEM_ID INTEGER,
         ITEM_TYPE INTEGER, UPDATE_PERIOD INTEGER, SUM INTEGER DEFAULT -1,
         FUZZY_NUM_VIEWS INTEGER DEFAULT -1)");
@@ -1559,3 +1558,12 @@ function upgradeDatabaseVersion54(&$db)
             $db->execute($sql);
         }
 }
+/**
+ * Upgrades a Version 54 version of the Yioop! database to a Version 55 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion55(&$db)
+{
+    $db->execute("CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
+                POS VARCHAR, PRIMARY KEY(WORD, LOCALE))");
+}
diff --git a/src/library/summarizers/CentroidSummarizer.php b/src/library/summarizers/CentroidSummarizer.php
index 1c807d1f..fb1954e9 100644
--- a/src/library/summarizers/CentroidSummarizer.php
+++ b/src/library/summarizers/CentroidSummarizer.php
@@ -105,7 +105,6 @@ class CentroidSummarizer extends Summarizer
         } else {
             $sentences = $out_sentences;
         }
-print_r($sentences);
         /*  Splitting into terms */
         $terms = [];
         foreach ($sentences as $sentence) {
diff --git a/src/locale/en_US/resources/Tokenizer.php b/src/locale/en_US/resources/Tokenizer.php
index a2d611e6..9ed3045c 100755
--- a/src/locale/en_US/resources/Tokenizer.php
+++ b/src/locale/en_US/resources/Tokenizer.php
@@ -1147,7 +1147,7 @@ class Tokenizer
     public static function questionParser($question)
     {
         $tagged_question = self::tagTokenizePartOfSpeech($question);
-        $generated_question_array = [];
+        $generated_questions = [];
         if (isset($tagged_question[0])) {
             if (in_array(trim($tagged_question[0]['tag']),
                 ["WRB", "WP"])) {
diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index 771f1e3c..992573af 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -28,6 +28,7 @@
  */
 namespace seekquarry\yioop\locale\hi\resources;
 
+use seekquarry\yioop\models as M;
 /**
  * Hindi specific tokenization code. In particular, it has a stemmer,
  * The stemmer is my stab at porting Ljiljana Dolamic (University of Neuchatel,
@@ -42,30 +43,40 @@ namespace seekquarry\yioop\locale\hi\resources;
  */
 class Tokenizer
 {
+    public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"];
+
+    public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "PRP"];
+
+    public static $adjective_phrases = ["JJ", "JJR", "JJS"];
+
+    public static $postpositional_phrases = ["inj", "PREP", "proNN", "CONJ", 
+        "INT", "particle", "case", "PSP"];
+
+    public static $question_marker = "qqq";
     /**
-     * Words we don't want to be stemmed
-     * @var array
-     */
+    * Words we don't want to be stemmed
+    * @var array
+    */
     public static $no_stem_list = [];
     /**
-     * Stub function which could be used for a word segmenter.
-     * Such a segmenter on input thisisabunchofwords would output
-     * this is a bunch of words
-     *
-     * @param string $pre_segment  before segmentation
-     * @return string should return string with words separated by space
-     *     in this case does nothing
-     */
+    * Stub function which could be used for a word segmenter.
+    * Such a segmenter on input thisisabunchofwords would output
+    * this is a bunch of words
+    *
+    * @param string $pre_segment  before segmentation
+    * @return string should return string with words separated by space
+    *     in this case does nothing
+    */
     public static function segment($pre_segment)
     {
         return $pre_segment;
     }
     /**
-     * Removes the stop words from the page (used for Word Cloud generation)
-     *
-     * @param string $page the page to remove stop words from.
-     * @return string $page with no stop words
-     */
+    * Removes the stop words from the page (used for Word Cloud generation)
+    *
+    * @param string $page the page to remove stop words from.
+    * @return string $page with no stop words
+    */
     public static function stopwordsRemover($page)
     {
         $stop_words = [
@@ -95,11 +106,11 @@ class Tokenizer
         return $page;
     }
     /**
-     * Computes the stem of an Hindi word
-     *
-     * @param string $word the string to stem
-     * @return string the stem of $word
-     */
+    * Computes the stem of an Hindi word
+    *
+    * @param string $word the string to stem
+    * @return string the stem of $word
+    */
     public static function stem($word)
     {
         if (in_array($word, self::$no_stem_list)) {
@@ -109,11 +120,11 @@ class Tokenizer
         return $word;
     }
     /**
-     * Removes common Hindi suffixes
-     *
-     * @param string $word to remove suffixes from
-     * @return string result of suffix removal
-     */
+    * Removes common Hindi suffixes
+    *
+    * @param string $word to remove suffixes from
+    * @return string result of suffix removal
+    */
     private static function removeSuffix($word)
     {
         $length = mb_strlen($word);
@@ -143,14 +154,14 @@ class Tokenizer
         return $word;
     }
     /**
-     * The method takes as input a phrase and returns a string with each
-     * term tagged with a part of speech.
-     *
-     * @param string $phrase text to add parts speech tags to
-     * @param bool $with_tokens whether to include the terms and the tags
-     *      in the output string or just the part of speech tags
-     * @return string $tagged_phrase which is a string of format term~pos
-     */
+    * The method takes as input a phrase and returns a string with each
+    * term tagged with a part of speech.
+    *
+    * @param string $phrase text to add parts speech tags to
+    * @param bool $with_tokens whether to include the terms and the tags
+    *      in the output string or just the part of speech tags
+    * @return string $tagged_phrase which is a string of format term~pos
+    */
     public static function tagPartsOfSpeechPhrase($phrase, $with_tokens = true)
     {
         $tagged_tokens = self::tagTokenizePartOfSpeech($phrase);
@@ -159,152 +170,145 @@ class Tokenizer
         return $tagged_phrase;
     }
     /**
-     * Uses the lexicon to assign a tag to each token and then uses a rule
-     * based approach to assign the most likely of tags to each token
-     *
-     * @param string $text input phrase which is to be tagged
-     * @return string $result which is an array of token => tag
-     */
-    public static function tagTokenizePartsofSpeech($text)
+    * Uses the lexicon to assign a tag to each token and then uses a rule
+    * based approach to assign the most likely of tags to each token
+    *
+    * @param string $text input phrase which is to be tagged
+    * @return string $result which is an array of token => tag
+    */
+    public static function tagTokenizePartofSpeech($text)
     {
-        static $dictionary = [];
-        if (empty($dictionary)) {
-            $serial_lex_file = C\LOCALE_DIR .
-                "/hi/resources/serial_lexicon.txt";
-            if (file_exists($serial_lex_file)) {
-                $dictionary = unserialize(file_get_contents($serial_lex_file));
-            } else {
-                $lines = gzfile(C\LOCALE_DIR . "/hi/resources/lexicon.txt.gz");
-                foreach ($lines as $line) {
-                    $tags = explode(',', trim($line, ' '));
-                    $dictionary[array_shift($tags)] = $tags;
-                }
-                $dictionary_string = serialize($dictionary);
-                file_put_contents($serial_lex_file, $dictionary_string);
-                chmod(0777, $serial_lex_file);
-            }
-        }
-        preg_match_all("/[\w\d]+/", $text, $matches);
-        $nouns = ['NN','NNP','NNS'];
-        $verbs = ['VBZ','VBD','VBN'];
-        $tokens = explode(' ', $text);
+        $tokens = preg_split("/[\s]+/", $text);
         $result = [];
         $tag_list = [];
         $i = 0;
-        foreach ($tokens as $token) {
-            /**
-             * Tag the tokens as found in the Lexicon, else use default tag as
-             * UNKNOWN
-             */
+        $model = new M\Model();
+        foreach ($tokens as $token) 
+        {
+            //Tag the tokens as found in the Lexicon
+            $token = trim($token);
             $current = ['token' => $token, 'tag' => 'UNKNOWN'];
-            if (!empty($dictionary[$token])) {
-                $tag_list = $dictionary[$token];
-                $current['tag'] = $tag_list[0];
-            }
-            /**
-             * NOUN IDENTIFICATION
-             * RULE 1: If the previous word tagged is a Adjective / Pronoun /
-             * Postposition then the current word is likely to be a noun
-             */
-            if ($previous['tag'] == 'JJ' ||
-                $previous['tag'] == 'PRP' ||
-                $previous['tag'] == 'POST_POS') {
-                $current['tag'] = 'NN';
-            }
-            /**
-             * NOUN INDENTIFICATION
-             * RULE 2: If the current word is a verb then the previous word is
-             * likely to be a noun
-             */
-            if (in_array($current['tag'], $verbs)) {
-                $previous['tag'] = 'NN';
-                $result[$i-1] = $previous;
-            }
-            /**
-             * NOUN INDENTIFICATION
-             * RULE 3: If the current tag is a noun then next / previous is
-             * likely to be a noun
-             */
-            /**
-             * DEMONSTRATIVE IDENTIFICATION
-             * RULE 1: If the current and previous words are tagged as pronouns
-             * then the previous word is likley to be a demonstrative
-             */
-            if ($current['tag'] == 'PRP' && $previous['tag'] == 'PRP') {
-                $previous['tag'] = 'DEM';
-                $result[$i-1] = $previous;
+            $word = $current['token'];
+            $sql = "SELECT * FROM LEXICON WHERE WORD = '{$word}' 
+                AND LOCALE = 'hi'";
+            $queryResult = @$model->db->execute($sql);     
+            if ($queryResult !== false) {
+                $row = $model->db->fetchArray($queryResult);
+                $current['tag'] = $row['POS'];
             }
-            /**
-             * DEMONSTRATIVE IDENTIFICATION
-             * RULE 2: If current word is a noun and previous word is a pronoun
-             * then the current word is liklely to be demonstrative
-             */
-            if ($current['tag'] == 'NN' && $previous['tag'] == 'PRP') {
-                $current['tag'] = 'DEM';
-            }
-            /**
-             * PRONOUN IDENTIFICATION
-             * RULE 1: If the previous word is unknown and cuurent word is a
-             * noun then the previous word is most likely to be a pronoun
-             */
-            if ($previous['tag'] == 'UNKNOWN' && $current['tag'] == 'NN') {
-                $previous['tag'] = 'PRP';
-                $result[$i - 1] = $previous;
-            }
-            /**
-             * NAME Identification
-             * RULE: If we get two words which are untagged the most probably
-             * they form a name and will be tagged as noun
-             */
-            if ($previous['tag'] == 'UNKNOWN' && $current['tag'] == 'UNKNOWN') {
-                $current['tag'] = 'NN';
-                $previous['tag'] = 'NN';
-                $result[$i - 1] = $previous;
+
+            if (is_numeric($token)) {
+                $current['tag'] = "NN";
+            } else if (strcmp($token,"है") == 0 || 
+                        strcmp($token, "हैं") == 0) {
+                $current['tag'] = "VB";
             }
-           /**
-            * ADJECTIVE IDENTIFCATION
-            * RULE: If the word ends with <tar>, <tam>, <thik> then we tag it
-            * as a Adjective
-            */
-            /**
-             * VERB IDENTIFICATION
-             * RULE: If the current word is tagged as Auxilary verb and
-             * previous word is tagged as Unknown then most likely that the
-             * previous word is a verb
-             */
-            if ($current['tag'] == 'VAUX' && $previous['tag'] == 'UNKNOWN') {
-                $previous['tag'] = 'VB';
-                $result[$i - 1] = $previous;
+
+            if (!isset($current['tag'])) {
+                $current['tag'] = "UNKNOWN";
             }
+
             $result[$i] = $current;
             $i++;
+        }
+        return self::tagUnknownWords($result);
+    }
+    /**
+    *    This method tags the remaining words from the text.
+    */
+    public static function tagUnknownWords($partiallyTaggedText) 
+    {
+        $result = $partiallyTaggedText;
+        $verbs = ['VBZ','VBD','VBN'];
+        $length = count($result);
+        $previous = $result[0];
+        for ($i = 1; $i < $length; $i++)
+        {
+            $current = $result[$i];
+            $current['token'] = trim($current['token']);
+            $current['tag'] = trim($current['tag']);
+            if ($current['tag'] == "UNKNOWN" || $previous['tag'] == "UNKNOWN") 
+            {
+                //RULE 1: If the previous word tagged is a Adjective / Pronoun 
+                // Postposition then the current word is likely to be a noun
+                if ($previous['tag'] == 'JJ'     ||
+                    $previous['tag'] == 'PRO_NN' ||
+                    $previous['tag'] == 'POST_POS') {
+                    $current['tag'] = 'NN';
+                    $result[$i] = $current;
+                }
+                //RULE 2: If the current word is a verb then the previous word is 
+                //likely to be a noun
+                if (in_array($current['tag'], $verbs)) {
+                    $previous['tag'] = 'NN';
+                    $result[$i] = $previous;
+                }
+                //PRONOUN IDENTIFICATION
+                //RULE 3: If the previous word is unknown and cuurent word is a
+                //noun then the previous word is most likely to be a pronoun
+                if ($previous['tag'] == 'UNKNOWN' &&
+                    $current['tag'] == 'NN') {
+                    $previous['tag'] = 'PRP';
+                    $result[$i-1] = $previous;
+                }
+                //VERB IDENTIFICATION
+                //RULE 4: If the current word is tagged as Auxilary verb  and 
+                //previous word is tagged as Unknown then most likely that the 
+                //previous word is a verb
+                if ($current['tag'] == 'VAUX' &&
+                    $previous['tag'] == 'UNKNOWN') {
+                    $previous['tag'] = 'VB';
+                    $result[$i-1] = $previous;
+                } 
+                //ADJECTIVE IDENTIFIATION
+                //RULE 5: if the currennt word ends with 'तम' or 'इक' or 'िक' 
+                //or 'तर' then the word is an adjective
+                if(mb_substr($current['token'], -2, 2) == "इक" || 
+                       mb_substr($current['token'], -2, 2) == "िक" ||
+                       mb_substr($current['token'], -2, 2) == "तर"  ||
+                       mb_substr($current['token'], -2, 2) == "तम") {
+                    $current['tag'] = 'AJ';
+                    $result[$i] = $current;
+                } 
+
+                if ($current['tag'] == "UNKNOWN") {
+                    $current['tag'] = 'NN';
+                    $result[$i] = $current;
+                } 
+
+                if ($previous['tag'] == "UNKNOWN"){
+                    $previous['tag'] = 'NN';
+                    $result[$i-1] = $previous; 
+                }
+            }
             $previous = $current;
-            $previous_token = $token;
         }
         return $result;
-    }
+     }
     /**
-     * This method is used to simplify the different tags of speech to a
-     * common form
-     *
-     * @param array $tagged_tokens which is an array of tokens assigned tags.
-     * @param bool $with_tokens whether to include the terms and the tags
-     *      in the output string or just the part of speech tags
-     * @return string $tagged_phrase which is a string fo form token~pos
-     */
+    * This method is used to simplify the different tags of speech to a
+    * common form
+    *
+    * @param array $tagged_tokens which is an array of tokens assigned tags.
+    * @param bool $with_tokens whether to include the terms and the tags
+    *      in the output string or just the part of speech tags
+    * @return string $tagged_phrase which is a string fo form token~pos
+    */
     public static function taggedPartOfSpeechTokensToString($tagged_tokens,
         $with_tokens = true)
     {
-        $tagged_phrase = [];
+        $tagged_phrase = "";
         $with_tokens = $with_tokens;
         $simplified_parts_of_speech = [
-          "NN" => "NN", "NNS" => "NN", "NNP" => "NN", "NNPS" => "NN",
-          "PRP" => "NN", 'PRP$' => "NN", "WP" => "NN",
+          "NNS" => "NN", "NNP" => "NN", "NNPS" => "NN","WP" => "NN",
           "VB" => "VB", "VBD" => "VB", "VBN" => "VB", "VBP" => "VB",
           "VBZ" => "VB",
           "JJ" => "AJ", "JJR" => "AJ", "JJS" => "AJ",
-          "RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV"
-        ];
+          "RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV",
+          "inj" => "IN", "case" => "IN", "proNN" => "IN", "particle" => "IN", 
+          "PREP" => "IN", "IN" => "IN", "PSP" => "IN",   
+          "direct_DT" => "DT",
+       ];
         foreach ($tagged_tokens as $t) {
             $tag = trim($t['tag']);
             $tag = (isset($simplified_parts_of_speech[$tag])) ?
@@ -314,4 +318,520 @@ class Tokenizer
         }
         return $tagged_phrase;
     }
+    /**
+    * Takes a part-of-speech tagged phrase and pre-tree with a
+    * parse-from position and builds a parse tree for a noun if possible
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @param array $tree that consists of ["curnode" =>
+    *      current parse position in $tagged_phrase]
+    * @return array has fields
+    *      "cur_node" index of how far we parsed $tagged_phrase
+    *      "NN" a subarray with a token node for the noun string that was
+    *      parsed
+    */
+    public static function extractNoun($tagged_phrase, $tree)
+    {
+        //Combining multiple noun into one
+        $noun_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            (in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$noun_phrases))) {
+            $noun_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($noun_string)) {
+            $tree["NN"] = $noun_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+    * Takes a part-of-speech tagged phrase and pre-tree with a
+    * parse-from position and builds a parse tree for a sequence of
+    * postpositional phrases if possible
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @param array $tree that consists of ["cur_node" =>
+    *      current parse position in $tagged_phrase]
+    * @return array has fields
+    *      "cur_node" index of how far we parsed $tagged_phrase
+    */
+    public static function extractPostposition($tagged_phrase, $tree, 
+        $index = 1)
+    {
+        $cur_node = $tree['cur_node'];
+        if (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$postpositional_phrases)) {
+            $preposition_string = "";
+            while (isset($tagged_phrase[$cur_node]['tag']) &&
+                in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$postpositional_phrases)) {
+                $preposition_string .= " ". $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            if (!empty($preposition_string)) {
+                $tree["IN_$index"] = $preposition_string;
+            }
+            if (isset($tagged_phrase[$cur_node]['tag']) &&
+                trim($tagged_phrase[$cur_node]['tag']) == "DT") {
+                $tree['DT_$index'] = $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            $adjective_string = "";
+            while (isset($tagged_phrase[$cur_node]['tag']) &&
+                in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$adjective_phrases)) {
+                $adjective_string .= " " . $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            if (!empty($adjective_string)) {
+                $tree["JJ_$index"] = $adjective_string;
+            }
+            $prep_noun_string = "";
+            while (isset($tagged_phrase[$cur_node]['tag']) &&
+                in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$noun_phrases)) {
+                $prep_noun_string .= " ". $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            if ($prep_noun_string) {
+                $tree["NP_$index"] = $prep_noun_string;
+            }
+            $tree_next = self::extractPostposition($tagged_phrase,
+                    ["cur_node" => $cur_node], $index + 1);
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+    * Takes a part-of-speech tagged phrase and pre-tree with a
+    * parse-from position and builds a parse tree for a noun phrase if possible
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @param array $tree that consists of ["curnode" =>
+    *      current parse position in $tagged_phrase]
+    * @return array has fields
+    *      "cur_node" index of how far we parsed $tagged_phrase
+    *      "JJ" with value an adjective subtree
+    *       "POST" with value a post position subtree
+    */
+    public static function extractNounPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_jj = self::extractAdjective($tagged_phrase, 
+            ['cur_node' => $tree['cur_node']]);
+         $tree_nn =self::extractNoun($tagged_phrase, 
+            ['cur_node' => $tree_jj['cur_node']]);
+        $tree_post = self::extractPostposition($tagged_phrase, 
+            ['cur_node' => $tree_nn['cur_node']]);
+        if ($tree_nn['cur_node'] == $cur_node) {
+            $tree['NP'] = "";
+        } else {
+            $cur_node = $tree_post['cur_node'];
+            unset($tree_jj['cur_node']);
+            $tree_new_sub['JJ'] = $tree_jj;
+            unset($tree_nn['cur_node']);
+            $tree_new_sub['NN'] = $tree_nn;
+            unset($tree_post['cur_node']);
+            $tree_new_sub['POST'] = $tree_post;
+            $tree_new['cur_node'] = $cur_node;
+            $tree_new['NP'] = $tree_new_sub;
+            return $tree_new;
+        }
+        return $tree;
+    }
+    /**
+    * Takes a part-of-speech tagged phrase and pre-tree with a
+    * parse-from position and builds a parse tree for a verb if possible
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @param array $tree that consists of ["curnode" =>
+    *      current parse position in $tagged_phrase]
+    * @return array has fields
+    *      "cur_node" index of how far we parsed $tagged_phrase
+    *      "VB" a subarray with a token node for the verb string that was
+    *      parsed
+    */
+    public static function extractVerb($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        // skip stuff before verb (intensifiers and adverbs)
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            !in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$verb_phrases)) {
+            $cur_node++;
+        }
+        $verb_string = "";
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$verb_phrases)) {
+            $verb_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($verb_string)) {
+            $tree["VB"] = $verb_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+    * Takes a part-of-speech tagged phrase and pre-tree with a
+    * parse-from position and builds a parse tree for a verb phrase if possible
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @param array $tree that consists of ["curnode" =>
+    *      current parse position in $tagged_phrase]
+    * @return array has fields
+    *      "cur_node" index of how far we parsed $tagged_phrase
+    *      "VP" a subarray with possible fields
+    *      "VB" with value a verb subtree
+    */
+    public static function extractVerbPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_vb = self::extractVerb($tagged_phrase, ['cur_node' => $cur_node]);
+        if ($tree_vb['cur_node'] == $cur_node) {
+            return $tree;
+        }
+        $cur_node = $tree_vb['cur_node'];
+        $preposition_string = "";
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']), 
+                self::$postpositional_phrases)) {
+            $preposition_string .= " ". $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($preposition_string)) {
+            $tree_vb["IN"] = $preposition_string;
+        }
+        $tree_np = self::extractNounPhrase($tagged_phrase,
+            ['cur_node' => $cur_node]);
+        $tree_new = [];
+        $tree_new_sub = [];
+        if ($tree_np['cur_node'] !=  $cur_node) {
+            $cur_node = $tree_np['cur_node'];
+            unset($tree_vb['cur_node'], $tree_np['cur_node']);
+            $tree_new_sub['VB'] = $tree_vb;
+            $tree_new_sub['NP'] = $tree_np['NP'];
+            $tree_new['cur_node'] = $cur_node;
+            $tree_new['VP'] = $tree_new_sub;
+            return $tree_new;
+        }
+        unset($tree_vb['cur_node']);
+        $tree_new_sub['VB'] = $tree_vb;
+        $tree_new['cur_node'] = $cur_node;
+        $tree_new['VP'] = $tree_new_sub;
+        return $tree_new;
+    }
+    /**
+    * Takes a part-of-speech tagged phrase and pre-tree with a
+    * parse-from position and builds a parse tree for an adjective if possible
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @param array $tree that consists of ["cur_node" =>
+    *      current parse position in $tagged_phrase]
+    * @return array has fields
+    *      "cur_node" index of how far we parsed $tagged_phrase
+    *      "JJ" a subarray with a token node for the adjective that was
+    *      parsed
+    */
+    public static function extractAdjective($tagged_phrase, $tree)
+    {
+        $adjective_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$adjective_phrases)) {
+            $adjective_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($adjective_string)) {
+            $tree["JJ"] = $adjective_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+    * Given a part-of-speeech tagged phrase array generates a parse tree
+    * for the phrase using a recursive descent parser.
+    *
+    * @param array $tagged_phrase
+    *      an array of pairs of the form ("token" => token_for_term,
+    *     "tag"=> part_of_speech_tag_for_term)
+    * @return array used to represent a tree. The array has up to three fields
+    *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
+    *      $tree["NP"] contains a subtree for a noun phrase
+    *      $tree["VP"] contains a subtree for a verb phrase
+    */
+    public static function generatePhraseParseTree($tagged_phrase)
+    {
+        $tree = [];
+        $tree_np = self::extractNounPhrase($tagged_phrase,['cur_node' => 0]);
+        $tree = ["cur_node" => $tree_np['cur_node']];
+        $tree_vp = self::extractVerbPhrase($tagged_phrase, $tree);
+        $tree['cur_node'] = $tree_vp['cur_node'];
+        if ($tree == $tree_vp) {
+            unset($tree_np['cur_node'], $tree_vp['cur_node']);
+            $tree['NP'] = $tree_np['NP'];
+            return $tree;
+        }
+        unset($tree_np['cur_node'], $tree_vp['cur_node']);
+        $tree['NP'] = $tree_np['NP'];
+        $tree['VP'] = $tree_vp['VP'];
+        return $tree;
+    }
+    /**
+    * Scans a word list for phrases. For phrases found generate
+    * a list of question and answer pairs at two levels of granularity:
+    * CONCISE (using all terms in orginal phrase) and RAW (removing
+    * (adjectives, etc).
+    *
+    * @param array $word_and_phrase_list of statements
+    * @return array with two fields: QUESTION_LIST consisting of
+    *      (SUBJECT, COMPLEMENT) where one of the components has been
+    *      replaced with a question marker.
+    */
+    public static function extractTripletsPhrases($word_and_phrase_list)
+    {
+        $triplets_list = [];
+        $question_list = [];
+        $question_answer_list = [];
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
+            $tagged_phrase = self::tagTokenizePartOfSpeech($word_and_phrase);
+            $parse_tree = self::generatePhraseParseTree($tagged_phrase);
+            $triplets = self::extractTripletsParseTree($parse_tree);
+            $extracted_triplets = self::rearrangeTripletsByType($triplets);
+            foreach ($triplet_types as $type) {
+                if (!empty($extracted_triplets[$type])) {
+                    $triplets = $extracted_triplets[$type];
+                    $questions = $triplets['QUESTION_LIST'];
+                    foreach ($questions as $question) {
+                        $question_list[$question] = $position_list;
+                    }
+                    $question_answer_list = array_merge($question_answer_list,
+                        $triplets['QUESTION_ANSWER_LIST']);
+                }
+            }
+        }
+        $out_triplets['QUESTION_LIST'] = $question_list;
+        $out_triplets['QUESTION_ANSWER_LIST'] = $question_answer_list;
+        return $out_triplets;
+    }
+    public static function extractDeepestSpeechPartPhrase($tree, $pos)
+    {
+        $extract = "";
+        if (!empty($tree[$pos])) {
+            $extract = self::extractDeepestSpeechPartPhrase($tree[$pos], $pos);
+        }
+        if (!$extract && !empty($tree[$pos]) && !empty($tree[$pos][$pos])) {
+            $extract = $tree[$pos][$pos];
+        }
+        return $extract;
+    }
+    public static function extractSubjectParseTree($tree)
+    {
+        $subject = [];
+        if (!empty($tree['NP'])) {
+            $subject['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree['NP'], "NN");
+            $raw_subject = "";
+            $it = new \RecursiveIteratorIterator(
+                new \RecursiveArrayIterator($tree['NP']));
+            foreach ($it as $v) {
+                $raw_subject .= $v . " ";
+            }
+            $subject['RAW']= $raw_subject;
+        } else {
+            $subject['CONCISE'] = "";
+            $subject['RAW'] = "";
+        }
+        return $subject;
+    }
+    public static function extractPredicateParseTree($tree)
+    {
+        $predicate = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            $predicate['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree_vp, "VB")
+            $raw_predicate = "";
+            if (!empty($tree_vp['VB'])) {
+                $tree_vb = $tree_vp['VB'];
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($tree_vb));
+                foreach ($it as $v) {
+                    $raw_predicate .= $v . " ";
+                }
+                $predicate['RAW'] = $raw_predicate;
+            }
+        } else {
+            $predicate['CONCISE'] = "";
+            $predicate['RAW'] = "";
+        }
+        return $predicate;
+    }
+    public static function extractObjectParseTree($tree)
+    {
+        $object = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            if (!empty($tree_vp['NP'])) {
+                $nb = $tree_vp['NP'];
+                $object['CONCISE'] = self::extractDeepestSpeechPartPhrase($nb,
+                    "NN");
+                $raw_object = "";
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($nb));
+                foreach ($it as $v) {
+                    $raw_object .= $v . " ";
+                }
+                $object['RAW'] = $raw_object;
+            } else {
+                $object['CONCISE'] = "";
+                $object['RAW'] = "";
+            }
+        } else {
+            $object['CONCISE'] = "";
+            $object['RAW'] = "";
+        }
+        return $object;
+    }
+    /**
+    * Takes a parse tree of a phrase and computes subject, predicate, and
+    * object arrays. Each of these array consists of two components CONCISE and
+    * RAW, CONCISE corresponding to something more similar to the words in the
+    * original phrase and RAW to the case where extraneous words have been
+    * removed
+    *
+    * @param are $tree a parse tree for a sentence
+    * @return array triplet array
+    */
+    public static function extractTripletsParseTree($parse_tree)
+    {
+        $triplets = [];
+        $triplets['subject'] = self::extractSubjectParseTree($parse_tree);
+        $triplets['object'] = self::extractObjectParseTree($parse_tree);
+        $triplets['predicate'] = self::extractPredicateParseTree($parse_tree);
+        return $triplets;
+    }
+    public static function rearrangeTripletsByType($sub_pred_obj_triplets)
+    {
+        $processed_triplet = [];
+        $processed_triplets['CONCISE'] =
+            self::extractTripletByType($sub_pred_obj_triplets, 'CONCISE');
+        $processed_triplets['RAW'] =
+            self::extractTripletByType($sub_pred_obj_triplets, 'RAW');
+        return $processed_triplets;
+    }
+    public static function extractTripletByType($sub_pred_obj_triplets, $type)
+    {
+        $triplets = [];
+        if (!empty($sub_pred_obj_triplets['subject'])
+            && !empty($sub_pred_obj_triplets['predicate'])
+            && !empty($sub_pred_obj_triplets['object'])) {
+            $question_answer_triplets = [];
+            $question_marker = self::$question_marker;
+            $sentence = [$sub_pred_obj_triplets['subject'][$type],
+                    $sub_pred_obj_triplets['predicate'][$type],
+                    $sub_pred_obj_triplets['object'][$type]];
+            $question_triplets = [];
+            for ($j = 0; $j < 2; $j++) {
+                for ($i = 0; $i < 3; $i++) {
+                    $q_sentence = $sentence;
+                    $q_sentence[$i] = $question_marker;
+                    $q_sentence_string = implode(" ", $q_sentence);
+                    $question_triplets[] = $q_sentence_string;
+                    $question_answer_triplets[$q_sentence_string] =
+                        preg_replace('/\s+/u', ' ',$sentence[$i]);
+                }
+            }
+            $triplets['QUESTION_LIST'] = $question_triplets;
+            $triplets['QUESTION_ANSWER_LIST'] = $question_answer_triplets;
+        }
+        return $triplets;
+    }
+    /**
+    * Takes tagged question string starts with Who
+    * and returns question triplet from the question string
+    *
+    * @param string $tagged_question part-of-speech tagged question
+    * @param int $index current index in statement
+    * @return array parsed triplet
+    */
+    public static function parseWhoQuestion($tagged_question, $index)
+    {
+        $generated_questions = [];
+        $question_marker = self::getQuestionMarker();
+        $triplets = [];
+        $tree_np = self::extractNounPhrase($tagged_question, ["cur_node" => 0]);
+        $triplets['subject'] = self::extractSubjectParseTree($tree_np);
+        $tree = ["cur_node" => $index];
+        $tree['NP'] = "कौन";
+        $tree_vp = self::extractVerbPhrase($tagged_question, $tree);
+        $triplets['predicate'] = self::extractPredicateParseTree($tree_vp);
+        $triplets['object'] = self::extractObjectParseTree($tree_vp);
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($triplet_types as $type) {
+            if (!empty($triplets['object'][$type])
+                && !empty($triplets['predicate'][$type])) {
+                $generated_questions[] = trim($triplets['object'][$type]) .
+                    " " . trim($triplets['predicate'][$type]) . " " .
+                    $question_marker;
+                $generated_questions[] = trim($triplets['object'][$type]) .
+                    " " . $question_marker .
+                    " " . trim($triplets['predicate'][$type]);
+            }
+        }
+        return $generated_questions;
+    }
+    /**
+    * Takes a phrase query entered by user and return true if it is question
+    * and false if not
+    *
+    * @param $phrase any statement
+    * @return bool returns true if statement is question
+    */
+    public function isQuestion($phrase)
+    {
+        $who_question = "कौन";
+        $phrase = trim($phrase);
+        if (mb_strpos($phrase, $who_question) !== false) {
+            return true;
+        }
+        return false;
+    }
+    public static function getQuestionMarker()
+    {
+        return self::$question_marker;
+    }
+    public static function questionParser($question)
+    {
+        $tagged_question = self::tagTokenizePartOfSpeech($question);
+        $generated_questions = "";
+        $index = -1;
+        foreach ($tagged_question as $key => $value) {
+            if (strcmp("कौन", $value['token']) == 0) {
+                $index = $key;
+                break;
+            }
+        }
+        $generated_questions = self::parseWhoQuestion($tagged_question, $index);
+        return $generated_questions;
+    }
 }
diff --git a/src/locale/hi/resources/lexicon.txt.gz b/src/locale/hi/resources/lexicon.txt.gz
index 5d7c3ba79dca9df5a33541c6a74cb8192e6cc93d..4f2e2010103459b68bb15a9725df198eda332dd7 100755
GIT binary patch
-- 
2.15.0.windows.1


From 91367d6881092eb00ab61ca7b23b6a7cf1d181fb Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Mon, 6 Nov 2017 19:37:43 -0800
Subject: [PATCH 2/2]  Hindi Triplet Extractor Changes for patch

---
 src/configs/Createdb.php              |   1 +
 src/library/PhraseParser.php          |   3 +
 src/locale/hi/resources/Tokenizer.php | 471 ++++++++++++++++++++--------------
 3 files changed, 285 insertions(+), 190 deletions(-)

diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index 3430501f..6c963789 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -39,6 +39,7 @@ use seekquarry\yioop\library\Cipher;
 use seekquarry\yioop\models\Model;
 use seekquarry\yioop\models\ProfileModel;
 use seekquarry\yioop\models\GroupModel;
+use seekquarry\yioop\configs as C;
 
 if (!empty($_SERVER['DOCUMENT_ROOT'])) {
     echo "BAD REQUEST";
diff --git a/src/library/PhraseParser.php b/src/library/PhraseParser.php
index ca8b23b5..9239badf 100755
--- a/src/library/PhraseParser.php
+++ b/src/library/PhraseParser.php
@@ -401,6 +401,9 @@ class PhraseParser
         } else {
             mb_internal_encoding("UTF-8");
             $string = mb_strtolower($string);
+            if ($lang == "hi") {
+                $string = preg_replace('/(,:)\p{P}/u', "", $string);
+            }
             $string = mb_ereg_replace("\s+|".C\PUNCT, " ", $string);
             $terms = self::segmentSegment($string, $lang);
             $terms = self::charGramTerms($terms, $lang);
diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index 992573af..f0df8568 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -43,40 +43,57 @@ use seekquarry\yioop\models as M;
  */
 class Tokenizer
 {
+    /**
+     * List of verb-like parts of speech that might appear in lexicon
+     * @array
+     */
     public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"];
-
+    /**
+     * List of noun-like parts of speech that might appear in lexicon
+     * @array
+     */
     public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "PRP"];
-
+    /**
+     * List of adjective-like parts of speech that might appear in lexicon
+     * @array
+     */
     public static $adjective_phrases = ["JJ", "JJR", "JJS"];
-
+    /**
+     * List of postpositional-like parts of speech that might appear in lexicon
+     * @array
+     */
     public static $postpositional_phrases = ["inj", "PREP", "proNN", "CONJ", 
         "INT", "particle", "case", "PSP"];
-
+    /**
+     * Any unique identifier corresponding to the component of a triplet which
+     * can be answered using a question answer list
+     * @string
+     */
     public static $question_marker = "qqq";
     /**
-    * Words we don't want to be stemmed
-    * @var array
-    */
+     * Words we don't want to be stemmed
+     * @var array
+     */
     public static $no_stem_list = [];
     /**
-    * Stub function which could be used for a word segmenter.
-    * Such a segmenter on input thisisabunchofwords would output
-    * this is a bunch of words
-    *
-    * @param string $pre_segment  before segmentation
-    * @return string should return string with words separated by space
-    *     in this case does nothing
-    */
+     * Stub function which could be used for a word segmenter.
+     * Such a segmenter on input thisisabunchofwords would output
+     * this is a bunch of words
+     *
+     * @param string $pre_segment  before segmentation
+     * @return string should return string with words separated by space
+     *     in this case does nothing
+     */
     public static function segment($pre_segment)
     {
         return $pre_segment;
     }
     /**
-    * Removes the stop words from the page (used for Word Cloud generation)
-    *
-    * @param string $page the page to remove stop words from.
-    * @return string $page with no stop words
-    */
+     * Removes the stop words from the page (used for Word Cloud generation)
+     *
+     * @param string $page the page to remove stop words from.
+     * @return string $page with no stop words
+     */
     public static function stopwordsRemover($page)
     {
         $stop_words = [
@@ -106,11 +123,11 @@ class Tokenizer
         return $page;
     }
     /**
-    * Computes the stem of an Hindi word
-    *
-    * @param string $word the string to stem
-    * @return string the stem of $word
-    */
+     * Computes the stem of an Hindi word
+     *
+     * @param string $word the string to stem
+     * @return string the stem of $word
+     */
     public static function stem($word)
     {
         if (in_array($word, self::$no_stem_list)) {
@@ -120,11 +137,11 @@ class Tokenizer
         return $word;
     }
     /**
-    * Removes common Hindi suffixes
-    *
-    * @param string $word to remove suffixes from
-    * @return string result of suffix removal
-    */
+     * Removes common Hindi suffixes
+     *
+     * @param string $word to remove suffixes from
+     * @return string result of suffix removal
+     */
     private static function removeSuffix($word)
     {
         $length = mb_strlen($word);
@@ -154,14 +171,14 @@ class Tokenizer
         return $word;
     }
     /**
-    * The method takes as input a phrase and returns a string with each
-    * term tagged with a part of speech.
-    *
-    * @param string $phrase text to add parts speech tags to
-    * @param bool $with_tokens whether to include the terms and the tags
-    *      in the output string or just the part of speech tags
-    * @return string $tagged_phrase which is a string of format term~pos
-    */
+     * The method takes as input a phrase and returns a string with each
+     * term tagged with a part of speech.
+     *
+     * @param string $phrase text to add parts speech tags to
+     * @param bool $with_tokens whether to include the terms and the tags
+     *      in the output string or just the part of speech tags
+     * @return string $tagged_phrase which is a string of format term~pos
+     */
     public static function tagPartsOfSpeechPhrase($phrase, $with_tokens = true)
     {
         $tagged_tokens = self::tagTokenizePartOfSpeech($phrase);
@@ -170,12 +187,12 @@ class Tokenizer
         return $tagged_phrase;
     }
     /**
-    * Uses the lexicon to assign a tag to each token and then uses a rule
-    * based approach to assign the most likely of tags to each token
-    *
-    * @param string $text input phrase which is to be tagged
-    * @return string $result which is an array of token => tag
-    */
+     * Uses the lexicon to assign a tag to each token and then uses a rule
+     * based approach to assign the most likely of tags to each token
+     *
+     * @param string $text input phrase which is to be tagged
+     * @return string $result which is an array of token => tag
+     */
     public static function tagTokenizePartofSpeech($text)
     {
         $tokens = preg_split("/[\s]+/", $text);
@@ -214,8 +231,8 @@ class Tokenizer
         return self::tagUnknownWords($result);
     }
     /**
-    *    This method tags the remaining words from the text.
-    */
+     *    This method tags the remaining words from the text.
+     */
     public static function tagUnknownWords($partiallyTaggedText) 
     {
         $result = $partiallyTaggedText;
@@ -286,14 +303,14 @@ class Tokenizer
         return $result;
      }
     /**
-    * This method is used to simplify the different tags of speech to a
-    * common form
-    *
-    * @param array $tagged_tokens which is an array of tokens assigned tags.
-    * @param bool $with_tokens whether to include the terms and the tags
-    *      in the output string or just the part of speech tags
-    * @return string $tagged_phrase which is a string fo form token~pos
-    */
+     * This method is used to simplify the different tags of speech to a
+     * common form
+     *
+     * @param array $tagged_tokens which is an array of tokens assigned tags.
+     * @param bool $with_tokens whether to include the terms and the tags
+     *      in the output string or just the part of speech tags
+     * @return string $tagged_phrase which is a string fo form token~pos
+     */
     public static function taggedPartOfSpeechTokensToString($tagged_tokens,
         $with_tokens = true)
     {
@@ -319,19 +336,19 @@ class Tokenizer
         return $tagged_phrase;
     }
     /**
-    * Takes a part-of-speech tagged phrase and pre-tree with a
-    * parse-from position and builds a parse tree for a noun if possible
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @param array $tree that consists of ["curnode" =>
-    *      current parse position in $tagged_phrase]
-    * @return array has fields
-    *      "cur_node" index of how far we parsed $tagged_phrase
-    *      "NN" a subarray with a token node for the noun string that was
-    *      parsed
-    */
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "NN" a subarray with a token node for the noun string that was
+     *      parsed
+     */
     public static function extractNoun($tagged_phrase, $tree)
     {
         //Combining multiple noun into one
@@ -350,18 +367,18 @@ class Tokenizer
         return $tree;
     }
     /**
-    * Takes a part-of-speech tagged phrase and pre-tree with a
-    * parse-from position and builds a parse tree for a sequence of
-    * postpositional phrases if possible
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @param array $tree that consists of ["cur_node" =>
-    *      current parse position in $tagged_phrase]
-    * @return array has fields
-    *      "cur_node" index of how far we parsed $tagged_phrase
-    */
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a sequence of
+     * postpositional phrases if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     */
     public static function extractPostposition($tagged_phrase, $tree, 
         $index = 1)
     {
@@ -411,19 +428,19 @@ class Tokenizer
         return $tree;
     }
     /**
-    * Takes a part-of-speech tagged phrase and pre-tree with a
-    * parse-from position and builds a parse tree for a noun phrase if possible
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @param array $tree that consists of ["curnode" =>
-    *      current parse position in $tagged_phrase]
-    * @return array has fields
-    *      "cur_node" index of how far we parsed $tagged_phrase
-    *      "JJ" with value an adjective subtree
-    *       "POST" with value a post position subtree
-    */
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" with value an adjective subtree
+     *       "POST" with value a post position subtree
+     */
     public static function extractNounPhrase($tagged_phrase, $tree)
     {
         $cur_node = $tree['cur_node'];
@@ -450,19 +467,19 @@ class Tokenizer
         return $tree;
     }
     /**
-    * Takes a part-of-speech tagged phrase and pre-tree with a
-    * parse-from position and builds a parse tree for a verb if possible
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @param array $tree that consists of ["curnode" =>
-    *      current parse position in $tagged_phrase]
-    * @return array has fields
-    *      "cur_node" index of how far we parsed $tagged_phrase
-    *      "VB" a subarray with a token node for the verb string that was
-    *      parsed
-    */
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VB" a subarray with a token node for the verb string that was
+     *      parsed
+     */
     public static function extractVerb($tagged_phrase, $tree)
     {
         $cur_node = $tree['cur_node'];
@@ -486,19 +503,19 @@ class Tokenizer
         return $tree;
     }
     /**
-    * Takes a part-of-speech tagged phrase and pre-tree with a
-    * parse-from position and builds a parse tree for a verb phrase if possible
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @param array $tree that consists of ["curnode" =>
-    *      current parse position in $tagged_phrase]
-    * @return array has fields
-    *      "cur_node" index of how far we parsed $tagged_phrase
-    *      "VP" a subarray with possible fields
-    *      "VB" with value a verb subtree
-    */
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VP" a subarray with possible fields
+     *      "VB" with value a verb subtree
+     */
     public static function extractVerbPhrase($tagged_phrase, $tree)
     {
         $cur_node = $tree['cur_node'];
@@ -537,19 +554,19 @@ class Tokenizer
         return $tree_new;
     }
     /**
-    * Takes a part-of-speech tagged phrase and pre-tree with a
-    * parse-from position and builds a parse tree for an adjective if possible
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @param array $tree that consists of ["cur_node" =>
-    *      current parse position in $tagged_phrase]
-    * @return array has fields
-    *      "cur_node" index of how far we parsed $tagged_phrase
-    *      "JJ" a subarray with a token node for the adjective that was
-    *      parsed
-    */
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for an adjective if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" a subarray with a token node for the adjective that was
+     *      parsed
+     */
     public static function extractAdjective($tagged_phrase, $tree)
     {
         $adjective_string = "";
@@ -567,17 +584,17 @@ class Tokenizer
         return $tree;
     }
     /**
-    * Given a part-of-speeech tagged phrase array generates a parse tree
-    * for the phrase using a recursive descent parser.
-    *
-    * @param array $tagged_phrase
-    *      an array of pairs of the form ("token" => token_for_term,
-    *     "tag"=> part_of_speech_tag_for_term)
-    * @return array used to represent a tree. The array has up to three fields
-    *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
-    *      $tree["NP"] contains a subtree for a noun phrase
-    *      $tree["VP"] contains a subtree for a verb phrase
-    */
+     * Given a part-of-speeech tagged phrase array generates a parse tree
+     * for the phrase using a recursive descent parser.
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @return array used to represent a tree. The array has up to three fields
+     *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
+     *      $tree["NP"] contains a subtree for a noun phrase
+     *      $tree["VP"] contains a subtree for a verb phrase
+     */
     public static function generatePhraseParseTree($tagged_phrase)
     {
         $tree = [];
@@ -596,16 +613,16 @@ class Tokenizer
         return $tree;
     }
     /**
-    * Scans a word list for phrases. For phrases found generate
-    * a list of question and answer pairs at two levels of granularity:
-    * CONCISE (using all terms in orginal phrase) and RAW (removing
-    * (adjectives, etc).
-    *
-    * @param array $word_and_phrase_list of statements
-    * @return array with two fields: QUESTION_LIST consisting of
-    *      (SUBJECT, COMPLEMENT) where one of the components has been
-    *      replaced with a question marker.
-    */
+     * Scans a word list for phrases. For phrases found generate
+     * a list of question and answer pairs at two levels of granularity:
+     * CONCISE (using all terms in orginal phrase) and RAW (removing
+     * (adjectives, etc).
+     *
+     * @param array $word_and_phrase_list of statements
+     * @return array with two fields: QUESTION_LIST consisting of
+     *      (SUBJECT, COMPLEMENT) where one of the components has been
+     *      replaced with a question marker.
+     */
     public static function extractTripletsPhrases($word_and_phrase_list)
     {
         $triplets_list = [];
@@ -633,6 +650,14 @@ class Tokenizer
         $out_triplets['QUESTION_ANSWER_LIST'] = $question_answer_list;
         return $out_triplets;
     }
+    /**
+     * Takes phrase tree $tree and a part-of-speech $pos returns
+     * the deepest $pos only path in tree.
+     *
+     * @param array $tree phrase to extract type from
+     * @param string $pos the part of speech to extract
+     * @return string the label of deepest $pos only path in $tree
+     */
     public static function extractDeepestSpeechPartPhrase($tree, $pos)
     {
         $extract = "";
@@ -644,6 +669,15 @@ class Tokenizer
         }
         return $extract;
     }
+    /**
+     * Takes a parse tree of a phrase or statement and returns an array
+     * with two fields CONCISE and RAW the former having the subject of
+     * the original phrase (as a string) the latter having the importart
+     * parts of the subject
+     *
+     * @param array representation of a parse tree of a phrase
+     * @return array with two fields CONCISE and RAW as described above
+     */
     public static function extractSubjectParseTree($tree)
     {
         $subject = [];
@@ -663,13 +697,22 @@ class Tokenizer
         }
         return $subject;
     }
+    /**
+     * Takes a parse tree of a phrase or statement and returns an array
+     * with two fields CONCISE and RAW the former having the predicate of
+     * the original phrase (as a string) the latter having the importart
+     * parts of the predicate
+     *
+     * @param array representation of a parse tree of a phrase
+     * @return array with two fields CONCISE and RAW as described above
+     */
     public static function extractPredicateParseTree($tree)
     {
         $predicate = [];
         if (!empty($tree['VP'])) {
             $tree_vp = $tree['VP'];
             $predicate['CONCISE'] = self::extractDeepestSpeechPartPhrase(
-                $tree_vp, "VB")
+                $tree_vp, "VB");
             $raw_predicate = "";
             if (!empty($tree_vp['VB'])) {
                 $tree_vb = $tree_vp['VB'];
@@ -686,6 +729,15 @@ class Tokenizer
         }
         return $predicate;
     }
+    /**
+     * Takes a parse tree of a phrase or statement and returns an array
+     * with two fields CONCISE and RAW the former having the object of
+     * the original phrase (as a string) the latter having the importart
+     * parts of the object
+     *
+     * @param array representation of a parse tree of a phrase
+     * @return array with two fields CONCISE and RAW as described above
+     */
     public static function extractObjectParseTree($tree)
     {
         $object = [];
@@ -713,15 +765,15 @@ class Tokenizer
         return $object;
     }
     /**
-    * Takes a parse tree of a phrase and computes subject, predicate, and
-    * object arrays. Each of these array consists of two components CONCISE and
-    * RAW, CONCISE corresponding to something more similar to the words in the
-    * original phrase and RAW to the case where extraneous words have been
-    * removed
-    *
-    * @param are $tree a parse tree for a sentence
-    * @return array triplet array
-    */
+     * Takes a parse tree of a phrase and computes subject, predicate, and
+     * object arrays. Each of these array consists of two components CONCISE and
+     * RAW, CONCISE corresponding to something more similar to the words in the
+     * original phrase and RAW to the case where extraneous words have been
+     * removed
+     *
+     * @param are $tree a parse tree for a sentence
+     * @return array triplet array
+     */
     public static function extractTripletsParseTree($parse_tree)
     {
         $triplets = [];
@@ -730,6 +782,15 @@ class Tokenizer
         $triplets['predicate'] = self::extractPredicateParseTree($parse_tree);
         return $triplets;
     }
+    /**
+     * Takes a triplets array with subject, predicate, object fields with
+     * CONCISE and RAW subfields and rearranges it to have two fields CONCISE
+     * and RAW with subject, predicate, object, and QUESTION_ANSWER_LIST
+     * subfields
+     *
+     * @param array $sub_pred_obj_triplets in format described above
+     * @return array $processed_triplets in format described above
+     */
     public static function rearrangeTripletsByType($sub_pred_obj_triplets)
     {
         $processed_triplet = [];
@@ -739,12 +800,22 @@ class Tokenizer
             self::extractTripletByType($sub_pred_obj_triplets, 'RAW');
         return $processed_triplets;
     }
+    /**
+     * Takes a triplets array with subject, predicate, object fields with
+     * CONCISE, RAW subfields and produces a triplits with $type subfield (where
+     * $type is one of CONCISE and RAW) and with subject, predicate, object,
+     * and QUESTION_ANSWER_LIST subfields
+     *
+     * @param array $sub_pred_obj_triplets  in format described above
+     * @param string $type either CONCISE or RAW
+     * @return array $triplets in format described above
+     */
     public static function extractTripletByType($sub_pred_obj_triplets, $type)
     {
         $triplets = [];
-        if (!empty($sub_pred_obj_triplets['subject'])
-            && !empty($sub_pred_obj_triplets['predicate'])
-            && !empty($sub_pred_obj_triplets['object'])) {
+        if (!empty($sub_pred_obj_triplets['subject'][$type])
+            && !empty($sub_pred_obj_triplets['predicate'][$type])
+            && !empty($sub_pred_obj_triplets['object'][$type])) {
             $question_answer_triplets = [];
             $question_marker = self::$question_marker;
             $sentence = [$sub_pred_obj_triplets['subject'][$type],
@@ -767,33 +838,38 @@ class Tokenizer
         return $triplets;
     }
     /**
-    * Takes tagged question string starts with Who
-    * and returns question triplet from the question string
-    *
-    * @param string $tagged_question part-of-speech tagged question
-    * @param int $index current index in statement
-    * @return array parsed triplet
-    */
+     * Takes tagged question string starts with Who
+     * and returns question triplet from the question string
+     *
+     * @param string $tagged_question part-of-speech tagged question
+     * @param int $index current index in statement
+     * @return array parsed triplet
+     */
     public static function parseWhoQuestion($tagged_question, $index)
     {
+        $start_pos = 0;
+        if ($index == 0)
+            $start_pos = $index + 1;
         $generated_questions = [];
         $question_marker = self::getQuestionMarker();
         $triplets = [];
-        $tree_np = self::extractNounPhrase($tagged_question, ["cur_node" => 0]);
+        $tree_np = self::extractNounPhrase($tagged_question, ["cur_node" => 
+            $start_pos]);
         $triplets['subject'] = self::extractSubjectParseTree($tree_np);
         $tree = ["cur_node" => $index];
-        $tree['NP'] = "कौन";
+        $tree['NP'] = $tagged_question[$index]['token'];
         $tree_vp = self::extractVerbPhrase($tagged_question, $tree);
         $triplets['predicate'] = self::extractPredicateParseTree($tree_vp);
-        $triplets['object'] = self::extractObjectParseTree($tree_vp);
         $triplet_types = ['CONCISE', 'RAW'];
         foreach ($triplet_types as $type) {
-            if (!empty($triplets['object'][$type])
+            if (!empty($triplets['subject'][$type])
                 && !empty($triplets['predicate'][$type])) {
-                $generated_questions[] = trim($triplets['object'][$type]) .
+                $generated_questions[$type][] = 
+                    trim($triplets['subject'][$type]) .
                     " " . trim($triplets['predicate'][$type]) . " " .
                     $question_marker;
-                $generated_questions[] = trim($triplets['object'][$type]) .
+                $generated_questions[$type][] = 
+                    trim($triplets['subject'][$type]) .
                     " " . $question_marker .
                     " " . trim($triplets['predicate'][$type]);
             }
@@ -801,12 +877,12 @@ class Tokenizer
         return $generated_questions;
     }
     /**
-    * Takes a phrase query entered by user and return true if it is question
-    * and false if not
-    *
-    * @param $phrase any statement
-    * @return bool returns true if statement is question
-    */
+     * Takes a phrase query entered by user and return true if it is question
+     * and false if not
+     *
+     * @param $phrase any statement
+     * @return bool returns true if statement is question
+     */
     public function isQuestion($phrase)
     {
         $who_question = "कौन";
@@ -816,22 +892,37 @@ class Tokenizer
         }
         return false;
     }
+    /**
+     * The function returns the question marker for the locale
+     *
+     * @return the question marker
+     */
     public static function getQuestionMarker()
     {
         return self::$question_marker;
     }
+    /**
+     * Takes WH questions and returns the triplet from the question
+     *
+     * @param string $question question to parse
+     * @return array question triplet
+     */
     public static function questionParser($question)
     {
+        /*
+         * Array of 'wh' questions: What, When, Where, Why, Who, Which, Whom, 
+         * Whose
+         */
+        $wh_questions = array( "क्या", "कब", "कहा", "क्यों", "कौन", "जिसे",
+            "जिसका", "कहाँ");
         $tagged_question = self::tagTokenizePartOfSpeech($question);
-        $generated_questions = "";
         $index = -1;
-        foreach ($tagged_question as $key => $value) {
-            if (strcmp("कौन", $value['token']) == 0) {
-                $index = $key;
+        foreach ($tagged_question as $i => $term_pos) {
+            if (in_array($term_pos['token'], $wh_questions)) {
+                $index = $i;
                 break;
             }
         }
-        $generated_questions = self::parseWhoQuestion($tagged_question, $index);
-        return $generated_questions;
+        return self::parseWhoQuestion($tagged_question, $index);
     }
 }
-- 
2.15.0.windows.1

