valeriocos · June 2, 2020 09:39 · Jun 2, 2020 · May 31, 2020 · May 31, 2020
diff --git a/migration68.py b/migration68.py
@@ -137,8 +137,8 @@ def main():
 
     dump(mapping_file_path, data_file_path)
     new_mapping_file_path = remove_release_date(mapping_file_path, ',\"release_date\":\{\"type\".\"date\"\}')
-    new_data_file_path = remove_release_date(data_file_path, ',\"release_date\":\".*\.[0-9]*\"',
-                                             fallback_reg_exp='\{\"release_date\":\".*\.[0-9]*\",')
+    new_data_file_path = remove_release_date(data_file_path, ',\"release_date\":\".*\.[0-9][0-9][0-9]*\"',
+                                             fallback_reg_exp='\{\"release_date\":\".*\.[0-9][0-9][0-9]*\",')
 
     print(DELETE_INDEX.format(USER, PWD, ENV))
 

diff --git a/migration68.py b/migration68.py
@@ -132,8 +132,8 @@ def main():
 
     os.mkdir(FOLDER_PATH)
 
-    mapping_file_path = FOLDER_PATH + ENV + '_mapping.json'
-    data_file_path = FOLDER_PATH + ENV + '.json'
+    mapping_file_path = FOLDER_PATH + 'kibana_mapping.json'
+    data_file_path = FOLDER_PATH + 'kibana.json'
 
     dump(mapping_file_path, data_file_path)
     new_mapping_file_path = remove_release_date(mapping_file_path, ',\"release_date\":\{\"type\".\"date\"\}')

diff --git a/migration68.py b/migration68.py
@@ -0,0 +1,157 @@
+import json
+import os
+import subprocess
+import re
+
+# TARGET ES
+ENV = 'localhost:9200'
+# CREDENTIALS
+USER = 'xxx'
+PWD = 'xxx'
+# PATH WHERE TO STORE AND MANIPULATE THE DATA
+FOLDER_PATH = '/home/kibana-migrations/'
+
+# DO NOT MODIFY AFTER HERE
+
+DUMP_MAPPING_TEMPLATE = '''
+NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
+  --input=https://{}:{}@{} \
+  --input-index=.kibana \
+  --output={} \
+  --limit=1000 \
+  --type=mapping
+'''
+
+DUMP_DATA_TEMPLATE = '''
+NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
+  --input=https://{}:{}@{} \
+  --input-index=.kibana \
+  --output={} \
+  --limit=1000 \
+  --type=data
+'''
+
+UPLOAD_MAPPING_TEMPLATE = '''
+NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
+  --input={} \
+  --output=https://{}:{}@{} \
+  --output-index=.kibana_old \
+  --limit=1000 \
+  --type=mapping
+'''
+
+UPLOAD_DATA_TEMPLATE = '''
+NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump \
+  --input={} \
+  --output=https://{}:{}@{} \
+  --output-index=.kibana_old \
+  --limit=1000 \
+  --type=data
+'''
+
+DELETE_INDEX = '''
+curl -XDELETE https://{}:{}@{}/.kibana -k
+'''
+
+SET_ALIAS_TEMPLATE = '''
+curl -XPOST https://{}:{}@{}/_aliases -d '{{"actions":[{{"add":{{"index":".kibana_old","alias":".kibana"}}}}]}}' -H 'Content-Type: application/json' -k
+'''
+
+SHOW_ALIASES = '''
+curl -XGET https://{}:{}@{}/_aliases?pretty -k
+'''
+
+
+def call(cmd):
+    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    for line in p.stdout.readlines():
+        print(line),
+    retval = p.wait()
+
+
+def dump(mapping_file_path, data_file_path):
+    cmd_dump_mapping = DUMP_MAPPING_TEMPLATE.format(USER, PWD, ENV, mapping_file_path)
+    call(cmd_dump_mapping)
+
+    cmd_dump_data = DUMP_DATA_TEMPLATE.format(USER, PWD, ENV, data_file_path)
+    call(cmd_dump_data)
+
+
+def remove_release_date(file_path, reg_exp, fallback_reg_exp=None):
+    lines = []
+    new_file_path = file_path.replace('.json', '_processed.json')
+    with open(file_path, 'r') as f:
+        content = f.readlines()
+        for line in content:
+            line_proc = line
+            if '\"release_date' in line:
+                line_proc = re.sub(reg_exp, '', line_proc)
+                if '\"release_date' in line_proc and fallback_reg_exp:
+                    line_proc = re.sub(fallback_reg_exp, '{', line_proc)
+                if '\"release_date' in line_proc:
+                    raise Exception
+
+                if 'mapping' in file_path:
+                    line_proc = line_proc.replace('{"doc":{"properties"', '{"doc":{"dynamic":"strict","properties"')
+                json.loads(line_proc)
+            lines.append(line_proc)
+
+    with open(new_file_path, 'w') as f:
+        for line in lines:
+            f.write(line)
+
+    return new_file_path
+
+
+def main():
+    """The script does the following:
+    - 1) create a local directory in your machine
+    - 2) dump in there the mappings and data from a target Kibana instance
+    - 3) process the files downloaded to remove the release_date and set the mappings to strict
+    - 4) print a set of commands to:
+      - a) delete the .kibana index
+      - b) upload the new mappings and data to the index .kibana_old in the target Kibana instance
+      - c) set the alias .kibana to the index .kibana_old
+      - d) check that the alias is set correctly
+
+    The workflow is the following:
+    - run the script
+    - check that the mappings and data have been correctly downloaded
+    - delete the current .kibana with 4a
+    - upload the .kibana_old generated with 4b
+    - set the alias .kibana to .kibana_old with 4c
+    - check that the alias is set correctly with 4d
+    - switch off the Kibana
+    - upgrade to Kibana 6.8
+
+    :return:
+    """
+    if os.path.isdir(FOLDER_PATH):
+        print("directory exists")
+        return
+
+    os.mkdir(FOLDER_PATH)
+
+    mapping_file_path = FOLDER_PATH + ENV + '_mapping.json'
+    data_file_path = FOLDER_PATH + ENV + '.json'
+
+    dump(mapping_file_path, data_file_path)
+    new_mapping_file_path = remove_release_date(mapping_file_path, ',\"release_date\":\{\"type\".\"date\"\}')
+    new_data_file_path = remove_release_date(data_file_path, ',\"release_date\":\".*\.[0-9]*\"',
+                                             fallback_reg_exp='\{\"release_date\":\".*\.[0-9]*\",')
+
+    print(DELETE_INDEX.format(USER, PWD, ENV))
+
+    cmd_upload_mapping = UPLOAD_MAPPING_TEMPLATE.format(new_mapping_file_path, USER, PWD, ENV)
+    print(cmd_upload_mapping)
+
+    cmd_dump_data = UPLOAD_DATA_TEMPLATE.format(new_data_file_path, USER, PWD, ENV)
+    print(cmd_dump_data)
+
+    print(SET_ALIAS_TEMPLATE.format(USER, PWD, ENV))
+
+    print(SHOW_ALIASES.format(USER, PWD, ENV))
+
+
+if __name__ == '__main__':
+    main()
No results found