move most of the updater settings to ini file

Also finally add the initial version of the upload scripts.

Change-Id: I3ad5bcbeba60f0cf9700e5fe5001a24f162a3244

Makefile.gbuild

@@ -35,8 +35,10 @@ create-update-info:
 	mkdir -p $(UPDATE_DIR)
 	mkdir -p $(MAR_DIR)/current-build
 	mkdir -p $(MAR_DIR)/language
-	MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/current-build/" "$(MAR_NAME_PREFIX)" "$(UPDATE_CERTIFICATE_NAME)" "$(UPDATE_CERTIFICATE_PATH)" "$(UPDATE_BASE_URL)"
-	MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/language" "$(MAR_NAME_PREFIX)" "$(UPDATE_CERTIFICATE_NAME)" "$(UPDATE_CERTIFICATE_PATH)" "$(UPDATE_BASE_URL)"
-	$(SRCDIR)/bin/update/create_build_config.py "$(PRODUCTNAME)" "$(VERSION)" "$(BUILDID)" "$(UPDATE_CHANNEL)" "$(PLATFORM)" "$(UPDATE_DIR)"
+	MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/current-build/" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)"
+	MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(UPDATE_DIR)" "$(MAR_DIR)/language" "$(MAR_NAME_PREFIX)" "$(UPDATE_CONFIG)"
+	$(SRCDIR)/bin/update/create_build_config.py "$(PRODUCTNAME)" "$(VERSION)" "$(BUILDID)" "$(PLATFORM)" "$(UPDATE_DIR)" "$(UPDATE_CONFIG)" "$(UPDATE_CONFIG)"
+	$(SRCDIR)/bin/update/upload_builds.py "$(PRODUCTNAME)" "$(BUILDID)" "$(PLATFORM)" "$(UPDATE_DIR)" "$(UPDATE_CONFIG)"
+	$(SRCDIR)/bin/update/upload_build_config.py "$(UPDATE_DIR)" "$(UPDATE_CONFIG)"
 
 # vim: set noet sw=4 ts=4:

bin/update/create_build_config.py

@@ -4,40 +4,39 @@ import json
 import sys
 import os
 
-def update_url(old_url, **kwargs):
-    new_url = old_url
-    for key, val in kwargs.items():
-        new_url = new_url.replace(key, val)
+from config import parse_config
 
-    return new_url
+from tools import replace_variables_in_string
 
 def update_all_url_entries(data, **kwargs):
-    data['complete']['url'] = update_url(data['complete']['url'], **kwargs)
+    data['complete']['url'] = replace_variables_in_string(data['complete']['url'], **kwargs)
     for language in data['languages']:
-        language['complete']['url'] = update_url(language['complete']['url'], **kwargs)
+        language['complete']['url'] = replace_variables_in_string(language['complete']['url'], **kwargs)
 
 def main(argv):
     if len(argv) < 7:
-        print("Usage: create_build_config.py $PRODUCTNAME $VERSION $BUILDID $UPDATECHANNEL $PLATFORM $TARGETDIR")
+        print("Usage: create_build_config.py $PRODUCTNAME $VERSION $BUILDID $PLATFORM $TARGETDIR $UPDATE_CONFIG")
+
+    config = parse_config(argv[6])
 
     data = { 'productName' : argv[1],
             'version' : argv[2],
             'buildNumber' : argv[3],
-            'updateChannel' : argv[4],
-            'platform' : argv[5]
+            'updateChannel' : config.channel,
+            'platform' : argv[4]
             }
 
     extra_data_files = ['complete_info.json', 'complete_lang_info.json']
 
     for extra_file in extra_data_files:
-        extra_file_path = os.path.join(argv[6], extra_file)
+        extra_file_path = os.path.join(argv[5], extra_file)
         with open(extra_file_path, "r") as f:
             extra_data = json.load(f)
             data.update(extra_data)
 
-    update_all_url_entries(data, channel=argv[4], platform=argv[5], buildid=argv[3], version=argv[2])
+    update_all_url_entries(data, channel=config.channel, platform=argv[4], buildid=argv[3], version=argv[2])
 
-    with open(os.path.join(argv[6], "build_config.json"), "w") as f:
+    with open(os.path.join(argv[5], "build_config.json"), "w") as f:
         json.dump(data, f, indent=4)
 
 if __name__ == "__main__":

bin/update/create_full_mar.py

@@ -6,6 +6,7 @@ import subprocess
 import json
 
 from tools import uncompress_file_to_dir, get_file_info
+from config import parse_config
 
 current_dir_path = os.path.dirname(os.path.realpath(__file__))
 
@@ -15,19 +16,19 @@ def make_mar_name(target_dir, filename_prefix):
 
 def main():
     print(sys.argv)
-    if len(sys.argv) < 9:
-        print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $BASE_URL")
+    if len(sys.argv) < 7:
+        print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $UPDATE_CONFIG")
         sys.exit(1)
 
-    url = sys.argv[8]
-    certificate_path = sys.argv[7]
-    certificate_name = sys.argv[6]
+    update_config = sys.argv[6]
     filename_prefix = sys.argv[5]
     temp_dir = sys.argv[4]
     target_dir = sys.argv[3]
     workdir = sys.argv[2]
     product_name = sys.argv[1]
 
+    config = parse_config(update_config)
+
     tar_dir = os.path.join(workdir, "installation", product_name, "archive", "install", "en-US")
     tar_file = os.path.join(tar_dir, os.listdir(tar_dir)[0])
 
@@ -42,11 +43,11 @@ def main():
     subprocess.call([os.path.join(current_dir_path, 'make_full_update.sh'), mar_file, uncompress_dir])
 
     signed_mar_file = make_mar_name(target_dir, filename_prefix + '_signed')
-    subprocess.call([mar_executable, '-C', target_dir, '-d', certificate_path, '-n', certificate_name, '-s', mar_file, signed_mar_file])
+    subprocess.call([mar_executable, '-C', target_dir, '-d', config.certificate_path, '-n', config.certificate_name, '-s', mar_file, signed_mar_file])
 
     os.rename(signed_mar_file, mar_file)
 
-    file_info = { 'complete' : get_file_info(mar_file, url) }
+    file_info = { 'complete' : get_file_info(mar_file, config.base_url) }
 
     with open(os.path.join(target_dir, 'complete_info.json'), "w") as complete_info_file:
         json.dump(file_info, complete_info_file, indent = 4)

bin/update/create_full_mar_for_languages.py

@@ -7,6 +7,8 @@ import json
 
 from tools import uncompress_file_to_dir, get_file_info
 
+from config import parse_config
+
 current_dir_path = os.path.dirname(os.path.realpath(__file__))
 
 def make_complete_mar_name(target_dir, filename_prefix, language):
@@ -21,19 +23,19 @@ def create_lang_infos(mar_file_name, language, url):
 
 def main():
     print(sys.argv)
-    if len(sys.argv) < 9:
-        print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $BASE_URL")
+    if len(sys.argv) < 7:
+        print("Usage: create_full_mar_for_languages.py $PRODUCTNAME $WORKDIR $TARGETDIR $TEMPDIR $FILENAMEPREFIX $UPDATE_CONFIG")
         sys.exit(1)
 
-    url = sys.argv[8]
-    certificate_path = sys.argv[7]
-    certificate_name = sys.argv[6]
+    update_config = sys.argv[6]
     filename_prefix = sys.argv[5]
     temp_dir = sys.argv[4]
     target_dir = sys.argv[3]
     workdir = sys.argv[2]
     product_name = sys.argv[1]
 
+    config = parse_config(update_config)
+
     mar_executable = os.environ.get('MAR', 'mar')
 
     language_pack_dir = os.path.join(workdir, "installation", product_name + "_languagepack", "archive", "install")
@@ -53,10 +55,10 @@ def main():
         subprocess.call([os.path.join(current_dir_path, 'make_full_update.sh'), mar_file_name, directory])
 
         signed_mar_file = make_complete_mar_name(target_dir, filename_prefix + '_signed', language)
-        subprocess.call([mar_executable, '-C', target_dir, '-d', certificate_path, '-n', certificate_name, '-s', mar_file_name, signed_mar_file])
+        subprocess.call([mar_executable, '-C', target_dir, '-d', config.certificate_path, '-n', config.certificate_name, '-s', mar_file_name, signed_mar_file])
         os.rename(signed_mar_file, mar_file_name)
 
-        lang_infos.append(create_lang_infos(mar_file_name, language, url))
+        lang_infos.append(create_lang_infos(mar_file_name, language, config.base_url))
 
     with open(os.path.join(target_dir, "complete_lang_info.json"), "w") as language_info_file:
         json.dump({'languages' : lang_infos}, language_info_file, indent=4)

bin/update/tools.py

@@ -42,3 +42,10 @@ def get_file_info(mar_file, url):
             'url' : url + os.path.basename(mar_file)}
 
     return data
+
+def replace_variables_in_string(string, **kwargs):
+    new_string = string
+    for key, val in kwargs.items():
+        new_string = new_string.replace('$(%s)'%key, val)
+
+    return new_string

bin/update/upload_build_config.py

+#! /usr/bin/env python3
+
+import sys
+import os
+import configparser
+import requests
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+def main(argv):
+
+    updater_config = sys.argv[2]
+
+    config = configparser.ConfigParser()
+    config.read(updater_config)
+
+    user = config["Updater"]["User"]
+    password = config["Updater"]["Password"]
+    base_address = config["Updater"]["ServerURL"]
+
+    login_url = base_address + "accounts/login/"
+
+    session = requests.session()
+    r1 = session.get(login_url)
+    csrftoken = session.cookies['csrftoken']
+
+    login_data = { 'username': user,'password': password,
+            'csrfmiddlewaretoken': csrftoken }
+    r1 = session.post(login_url, data=login_data, headers={"Referer": login_url})
+
+    url = base_address + "update/upload/release"
+
+    build_config = os.path.join(sys.argv[1], "build_config.json")
+    r = session.post(url, files={'release_config': open(build_config, "r")})
+    print(r.content)
+    if r.status_code != 200:
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main(sys.argv)

bin/update/upload_builds.py

+#! /usr/bin/env python3
+
+import sys
+import os
+import subprocess
+
+from config import parse_config
+
+from tools import replace_variables_in_string
+
+def main():
+    product_name = sys.argv[1]
+    buildid = sys.argv[2]
+    platform = sys.argv[3]
+    update_dir = sys.argv[4]
+    update_config = sys.argv[5]
+
+    config = parse_config(update_config)
+    upload_url = replace_variables_in_string(config.upload_url, channel=config.channel, buildid=buildid, platform=platform)
+
+    target_url, target_dir = upload_url.split(':')
+
+    subprocess.call(['ssh', target_url, "'mkdir -p %s'"%(target_dir)])
+    for file in os.listdir(update_dir):
+        if file.endswith('.mar'):
+            subprocess.call(['scp', os.path.join(update_dir, file), upload_url])
+
+if __name__ == '__main__':
+    main()

config_host.mk.in

@@ -575,12 +575,9 @@ export TOUCH=@TOUCH@
 export UCRTSDKDIR=@UCRTSDKDIR@
 export UCRTVERSION=@UCRTVERSION@
 export UNOWINREG_DLL=@UNOWINREG_DLL@
-export UPDATE_CERTIFICATE_NAME=@UPDATE_CERTIFICATE_NAME@
-export UPDATE_CERTIFICATE_PATH=@UPDATE_CERTIFICATE_PATH@
-export UPDATE_CHANNEL=@UPDATE_CHANNEL@
-export UPDATE_BASE_URL=@UPDATE_BASE_URL@
 export USE_LIBRARY_BIN_TAR=@USE_LIBRARY_BIN_TAR@
 export USE_XINERAMA=@USE_XINERAMA@
+export UPDATE_CONFIG=@UPDATE_CONFIG@
 export UUIDGEN=@UUIDGEN@
 export VALGRIND_CFLAGS=$(gb_SPACE)@VALGRIND_CFLAGS@
 export VCVER=@VCVER@

configure.ac

@@ -1383,26 +1383,9 @@ libo_FUZZ_ARG_ENABLE(online-update,
          enabled instead of the traditional update mechanism.]),
 ,)
 
-AC_ARG_WITH(update-channel,
-    AS_HELP_STRING([--with-update-channel=master-daily],
-        [Defines the update channel used by the updater.]),
-,with_update_channel=master-daily)
-
-AC_ARG_WITH(update-certificate-name,
-    AS_HELP_STRING([--with-update-certificate-name=master-daily],
-                   [Defines the name of the certificate that should be used to sign the mar files.
-                    This makes only sense if the onlineupdate is used and the mar files should be uploaded.]))
-
-AC_ARG_WITH(update-certificate-path,
-    AS_HELP_STRING([--with-update-certificate-name=master-daily],
-                   [Defines the path to the nss certificate store used for the updater
-                    This makes only sense if the onlineupdate is used and the mar files should be uploaded.]))
-
-AC_ARG_WITH(update-base-url,
-    AS_HELP_STRING([--with-update-base-url=http://dev-builds.libreoffice.org/update/$(channel)/$(buildid)],
-                    [Defines the base url for the mar file url. THe following variables are going to be replaced
-                     with the corresponding content: channel, buildid, version, platform, productname.
-                    This makes only sense if the onlineupdate is used and the mar files should be uploaded.]))
+AC_ARG_WITH(update-config,
+    AS_HELP_STRING([--with-update-config=/tmp/update.ini],
+                   [Path to the update config ini file]))
 
 libo_FUZZ_ARG_ENABLE(extension-update,
     AS_HELP_STRING([--disable-extension-update],
@@ -11324,10 +11307,7 @@ dnl ===================================================================
 AC_MSG_CHECKING([whether to enable online update])
 ENABLE_ONLINE_UPDATE=
 ENABLE_ONLINE_UPDATE_MAR=
-UPDATE_CHANNEL=
-UPDATE_CERTIFICATE_NAME=
-UPDATE_CERTIFICATE_PATH=
-UPDATE_BASE_URL=
+UPDATE_CONFIG=
 if test "$enable_online_update" = ""; then
     if test "$_os" = "WINNT" -o "$_os" = "Darwin"; then
         AC_MSG_RESULT([yes])
@@ -11339,11 +11319,8 @@ else
     if test "$enable_online_update" = "mar"; then
         AC_MSG_RESULT([yes - MAR-based online update])
         ENABLE_ONLINE_UPDATE_MAR="TRUE"
+        UPDATE_CONFIG="$with_update_config"
         AC_DEFINE(HAVE_FEATURE_UPDATE_MAR)
-        UPDATE_CHANNEL="$with_update_channel"
-        UPDATE_CERTIFICATE_NAME="$with_update_certificate_name"
-        UPDATE_CERTIFICATE_PATH="$with_update_certificate_path"
-        UPDATE_BASE_URL="$with_update_base_url"
     elif test "$enable_online_update" = "yes"; then
         AC_MSG_RESULT([yes])
         ENABLE_ONLINE_UPDATE="TRUE"
@@ -11353,10 +11330,7 @@ else
 fi
 AC_SUBST(ENABLE_ONLINE_UPDATE)
 AC_SUBST(ENABLE_ONLINE_UPDATE_MAR)
-AC_SUBST(UPDATE_CHANNEL)
-AC_SUBST(UPDATE_CERTIFICATE_NAME)
-AC_SUBST(UPDATE_CERTIFICATE_PATH)
-AC_SUBST(UPDATE_BASE_URL)
+AC_SUBST(UPDATE_CONFIG)
 
 dnl ===================================================================
 dnl Test whether we need bzip2