Merge pull request #12 from wikirate/update-entity-codenames-index-mapping

Update entity codenames index mapping

Author: AndrewKostka

config/mappings/companies.json

@@ -5,28 +5,12 @@
   "mappings": {
     "properties": {
       "headquarters": {
-        "type": "keyword"
-      },
-      "sec_cik": {
         "type": "text",
-        "analyzer": "keyword"
+        "index": "not_analyzed"
       },
-      "oar_id": {
-        "type": "text",
-        "analyzer": "keyword"
-      },
-      "open_corporates": {
-        "type": "text",
-        "analyzer": "keyword"
-      },
-      "uk_cn": {
-        "type": "text",
-        "analyzer": "keyword"
-      },
-      "aus_cn": {
-        "type":"text",
-        "analyzer": "keyword"
+      "coroporate_identifiers": {
+        "type": "keyword"
       }
     }
   }
-}
+}

config/queries/companies.sql

@@ -1,34 +1,19 @@
-SELECT
-  co.updated_at,
-  IF (
-    co.created_at = co.updated_at,
-    "create",
-    IF (co.trash = 1, "delete", "update")
-  ) AS action_type,
+SELECT co.updated_at,
+  IF(co.created_at = co.updated_at, "create", IF(co.trash=1, "delete", "update")) as action_type,
   co.id,
   co.name,
-  hq.db_content AS headquarters,
-  sec.db_content AS sec_cik,
-  oar.db_content AS oar_id,
-  oc.db_content AS open_corporates,
-  uk_cn.db_content AS uk_cn,
-  abn.db_content AS aus_cn
-FROM
-  cards AS co
-  LEFT JOIN cards AS hq ON co.id = hq.left_id
-  AND hq.right_id = :headquarters
-  LEFT JOIN cards AS sec ON co.id = sec.left_id
-  AND sec.right_id = :sec_cik
-  LEFT JOIN cards AS oar ON co.id = oar.left_id
-  AND oar.right_id = :oar_id
-  LEFT JOIN cards AS oc ON co.id = oc.left_id
-  AND oc.right_id = :open_corporates
-  LEFT JOIN cards AS uk_cn ON co.id = uk_cn.left_id
-  AND uk_cn.right_id = :uk_cn
-  LEFT JOIN cards AS abn ON co.id = abn.left_id
-  AND abn.right_id = :abn
-WHERE
-  co.type_id = :wikirate_company
-  AND co.updated_at > :sql_last_value
-ORDER BY
-  co.updated_at;
+  hq.db_content as headquarters,
+  GROUP_CONCAT(cids.db_content SEPARATOR ',') as corporate_identifiers
+FROM cards as co
+  LEFT JOIN cards as hq ON co.id = hq.left_id AND hq.right_id=:headquarters
+  LEFT JOIN cards as cids ON co.id=cids.left_id AND cids.right_id IN (SELECT id from cards where type_id=:company_identifier)
+WHERE co.type_id=:wikirate_company
+  AND co.updated_at >= :sql_last_value
+GROUP BY
+    co.updated_at,
+    co.created_at,
+    co.trash,
+    co.id,
+    co.name,
+    hq.db_content
+ORDER BY co.updated_at;

entrypoint.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 
 export DATABASE_PORT=${DATABASE_PORT:-3306}
 
-cards="'headquarters','sec_cik','oar_id','open_corporates_id','wikirate_company',\
-'metric','source','phrase','wikirate_title','wikirate_topic','project',\
-'research_group','company_group','dataset','uk_cn', 'abn'"
+cards="'headquarters','company',\
+'metric','source','phrase','wikirate_title','topic','project',\
+'research_group','company_group','dataset','company_identifier'"
 
 query="SELECT CONCAT('PARAM_', UPPER(IFNULL(codename, name)), '=', id) AS result \
 FROM cards WHERE codename IN ($cards) OR name IN ($cards)"

pipeline/wikirate.conf

@@ -8,8 +8,8 @@ input {
     last_run_metadata_path => "/usr/share/logstash/data/plugins/inputs/jdbc/logstash_jdbc_last_run_general"
     statement_filepath => "/usr/share/logstash/config/queries/general.sql"
     parameters => {
-      "wikirate_company" => "${PARAM_WIKIRATE_COMPANY:}"
-      "wikirate_topic" => "${PARAM_WIKIRATE_TOPIC:}"
+      "wikirate_company" => "${PARAM_COMPANY:}"
+      "wikirate_topic" => "${PARAM_TOPIC:}"
       "project" => "${PARAM_PROJECT:}"
       "research_group" => "${PARAM_RESEARCH_GROUP:}"
       "company_group" => "${PARAM_COMPANY_GROUP:}"
@@ -69,13 +69,10 @@ input {
     last_run_metadata_path => "/usr/share/logstash/data/plugins/inputs/jdbc/logstash_jdbc_last_run_companies"
     statement_filepath => "/usr/share/logstash/config/queries/companies.sql"
     parameters => {
+      "wikirate_company" => "${PARAM_COMPANY:}"
       "headquarters" => "${PARAM_HEADQUARTERS:}"
-      "sec_cik" => "${PARAM_SEC_CIK:}"
-      "oar_id" => "${PARAM_OAR_ID:}"
-      "open_corporates" => "${PARAM_OPEN_CORPORATES_ID:}"
-      "uk_cn" => "${PARAM_UK_CN:}"
-      "abn" => "${PARAM_ABN:}"
-      "wikirate_company" => "${PARAM_WIKIRATE_COMPANY:}"
+      "company_identifier" => "${PARAM_COMPANY_IDENTIFIER:}"
+      
     }
     sql_log_level => "debug"
     use_column_value => true