diff --git a/runbot/container.py b/runbot/container.py index 2fe340d9d..c82c3994b 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -261,8 +261,8 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False run_cmd = cmd run_cmd = f'cd /data/build;touch start-{container_name};{run_cmd};cd /data/build;touch end-{container_name}' run_cmd_repr = str(run_cmd) - if len(run_cmd_repr) > 250: - run_cmd_repr = run_cmd_repr[:250] + '...' + if len(run_cmd_repr) > 300: + run_cmd_repr = run_cmd_repr[:250] + '...' + run_cmd_repr[-50:] _logger.info('Docker run command: %s', run_cmd_repr) docker_clear_state(container_name, build_dir) # ensure that no state are remaining build_dir = file_path(build_dir) @@ -364,9 +364,6 @@ def docker_state(container_name, build_dir): if not exist: return 'VOID' - if os.path.exists(os.path.join(build_dir, f'end-{container_name}')): - return 'END' - state = 'UNKNOWN' if started: docker_client = docker.from_env() @@ -376,7 +373,6 @@ def docker_state(container_name, build_dir): state = 'RUNNING' if container.status in ('created', 'running', 'paused') else 'GHOST' except docker.errors.NotFound: state = 'GHOST' - # check if the end- file has been written in between time if state == 'GHOST' and os.path.exists(os.path.join(build_dir, f'end-{container_name}')): state = 'END' return state diff --git a/runbot/models/build.py b/runbot/models/build.py index ba08d8ee0..17beab77d 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -15,7 +15,6 @@ from dateutil import parser from markupsafe import Markup -from psycopg2 import sql from psycopg2.extensions import TransactionRollbackError from odoo import api, fields, models @@ -1206,15 +1205,6 @@ def _local_pg_dropdb(self, dbname): host_name = self.env['runbot.host']._get_current_name() self.env['runbot.runbot']._warning(f'Host {host_name}: {msg}') - def _local_pg_createdb(self, dbname): - icp = self.env['ir.config_parameter'] - db_template = icp.get_param('runbot.runbot_db_template', default='template0') - self._local_pg_dropdb(dbname) - _logger.info("createdb %s", dbname) - with local_pgadmin_cursor() as local_cr: - local_cr.execute(sql.SQL("""CREATE DATABASE {} TEMPLATE %s LC_COLLATE 'C' ENCODING 'unicode'""").format(sql.Identifier(dbname)), (db_template,)) - self.env['runbot.database'].create({'name': dbname, 'build_id': self.id}) - def _log(self, func, message, *args, level='INFO', log_type='runbot', path='runbot'): def truncate(message, maxlenght=300000): if len(message) > maxlenght: diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 2d0a70f3c..1a4de9581 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -52,6 +52,10 @@ PYTHON_DEFAULT = "# type python code here\n\n\n\n\n\n" +def echo(text): + return f'echo $(date -u "+%Y-%m-%d %H:%M:%S,%3N") "{text}"' + + def filter_all_modules(selector, build, dynamic_vars): if selector.split(',', 1)[0] != '*': selector = f'*,{selector}' @@ -428,8 +432,8 @@ class ConfigStep(models.Model): dockerfile_id = fields.Many2one('runbot.dockerfile', string='Dockerfile') dockerfile_variant = fields.Char('Docker Variant') # install_odoo - create_db = fields.Boolean('Create Db', default=True, tracking=True) # future - custom_db_name = fields.Char('Custom Db Name', tracking=True) # future + create_db = fields.Boolean('Create Db', default=True, tracking=True) # TODO remove + custom_db_name = fields.Char('Custom Db Name', tracking=True) install_modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='', tracking=True) db_name = fields.Char('Db Name', compute='_compute_db_name', inverse='_inverse_db_name', tracking=True) cpu_limit = fields.Integer('Cpu limit', default=3600, tracking=True) @@ -777,9 +781,8 @@ def _run_install_odoo(self, build, config_data=None): db_suffix = config_data.get('db_name') or (build.params_id.dump_db.db_suffix if not self.create_db else False) or self._get_db_name(build) db_suffix = re.sub(r'[^a-z0-9\-_]', '_', db_suffix.lower()) db_name = '%s-%s' % (build.dest, db_suffix) - if modules_to_install and self.create_db: - build._local_pg_createdb(db_name) cmd += ['-d', db_name] + self.env['runbot.database'].create({'name': db_name, 'build_id': build.id}) # Demo data behavior changed in 18.1 -> demo data became opt-in instead of opt-out available_options = build._parse_config() @@ -840,15 +843,18 @@ def _run_install_odoo(self, build, config_data=None): cmd.finals.extend(self._post_install_commands(build, config_data, py_version)) # coverage post, extra-checks, ... + env_variables = self.additionnal_env.split(';') if self.additionnal_env else [] + if config_env_variables := config_data.get('env_variables', False): + env_variables += config_env_variables.split(';') + if config_data.get('export_database', True): self._add_zip_generation(build, cmd, db_name) + env_variables.append('SAVE_CLUSTER=1') + env_variables.append(f'CLUSTER_BACKUP_NAME={db_name}-cluster.zip') if self.flamegraph: cmd.finals.append(['flamegraph.pl', '--title', 'Flamegraph %s for build %s' % (self.sanitized_name(build), build.id), self._perfs_data_path(build), '>', self._perfs_data_path(ext='svg')]) cmd.finals.append(['gzip', '-f', self._perfs_data_path(build)]) # keep data but gz them to save disc space - env_variables = self.additionnal_env.split(';') if self.additionnal_env else [] - if config_env_variables := config_data.get('env_variables', False): - env_variables += config_env_variables.split(';') if config_data.get('coverage_test_context', self.coverage_test_context): env_variables.append("COVERAGE_DYNAMIC_CONTEXT=test_function") @@ -864,9 +870,13 @@ def _add_zip_generation(self, build, cmd, db_name): filestore_path = '/data/build/datadir/filestore/%s' % db_name filestore_dest = '%s/filestore/' % dump_dir zip_path = '/data/build/logs/%s.zip' % db_name + cmd.finals.append([echo('### Generating dump')]) cmd.finals.append(['pg_dump', db_name, '>', sql_dest]) + cmd.finals.append([echo('### Copying filestore')]) cmd.finals.append(['cp', '-r', filestore_path, filestore_dest]) + cmd.finals.append([echo('### Generating zip')]) cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rmq9', zip_path, '*']) + cmd.finals.append([echo('### Done')]) infos = '{\n "db_name": "%s",\n "build_id": %s,\n "shas": [%s]\n}' % (db_name, build.id, ', '.join(['"%s"' % build_commit.commit_id.dname for build_commit in build.params_id.commit_link_ids])) build._write_file('logs/%s/info.json' % db_name, infos) @@ -1123,6 +1133,7 @@ def _run_restore(self, build, config_data=None): dump_db = params.dump_db if dump_url := config_data.get('dump_url'): zip_name = dump_url.split('/')[-1] + download_db_name = zip_name.replace('.zip', '') build._log('_run_restore', f'Restoring db [{zip_name}]({dump_url})', log_type='markdown') else: reference_build = None @@ -1167,28 +1178,48 @@ def _run_restore(self, build, config_data=None): target_suffix = config_data.get('target_suffix', self.restore_rename_db_suffix or download_db_suffix) restore_db_name = '%s-%s' % (build.dest, target_suffix) - build._local_pg_createdb(restore_db_name) - cmd = ' && '.join([ - 'mkdir /data/build/restore', - 'cd /data/build/restore', - 'wget --retry-on-host-error %s' % dump_url, - 'unzip -q %s' % zip_name, - 'echo "### restoring filestore"', - 'mkdir -p /data/build/datadir/filestore/%s' % restore_db_name, - 'mv filestore/* /data/build/datadir/filestore/%s' % restore_db_name, - 'echo "### restoring db"', - 'psql -q %s < dump.sql' % (restore_db_name), - 'echo "### performing an analyze"', - 'psql -q -d %s -c "ANALYZE;"' % restore_db_name, - 'cd /data/build', - 'echo "### cleaning"', - 'rm -r restore', - 'echo "### listing modules"', - """psql %s -c "select name from ir_module_module where state = 'installed'" -t -A > /data/build/logs/restore_modules_installed.txt""" % restore_db_name, - 'echo "### restore" "successful"', # two part string to avoid miss grep - ]) - - return dict(cmd=cmd, network_enabled=True) + icp = self.env['ir.config_parameter'] + db_template = icp.get_param('runbot.runbot_db_template', default='template0') + self.env['runbot.database'].create({'name': restore_db_name, 'build_id': build.id}) + restore_cluster = True # todo define is it is applicable + env_variables = [] + if restore_cluster: + cluster_zip_url = dump_url.replace('.zip', '-cluster.zip') + env_variables.append(f'RESTORE_ZIP_URL={cluster_zip_url}') + cmd = ' && '.join([ + echo('### Moving filestore'), + f'mv /data/build/datadir/filestore/{download_db_name} /data/build/datadir/filestore/{restore_db_name}', + echo('### Renaming db'), + f'psql -q -d postgres -c \'ALTER DATABASE "{download_db_name}" RENAME TO "{restore_db_name}";\'', + echo('### Performing an analyze'), + f'psql -q -d {restore_db_name} -c "ANALYZE;"', + f"""psql {restore_db_name} -c "select name from ir_module_module where state = 'installed'" -t -A > /data/build/logs/restore_modules_installed.txt""", + echo('### restore" "successful'), # two part string to avoid miss grep + ]) + else: + cmd = ' && '.join([ + 'mkdir /data/build/restore', + 'cd /data/build/restore', + echo('### getting archive'), + 'wget --retry-on-host-error %s' % dump_url, + 'unzip -q %s' % zip_name, + echo('### restoring filestore'), + 'mkdir -p /data/build/datadir/filestore/%s' % restore_db_name, + 'mv filestore/* /data/build/datadir/filestore/%s' % restore_db_name, + echo('### restoring db'), + 'createdb %s -T %s' % (restore_db_name, db_template), + 'psql -q %s < dump.sql' % (restore_db_name), + echo('### performing an analyze'), + 'psql -q -d %s -c "ANALYZE;"' % restore_db_name, + 'cd /data/build', + echo('### cleaning'), + 'rm -r restore', + echo('### listing modules'), + """psql %s -c "select name from ir_module_module where state = 'installed'" -t -A > /data/build/logs/restore_modules_installed.txt""" % restore_db_name, + echo('### restore" "successful'), # two part string to avoid miss grep + ]) + + return dict(cmd=cmd, network_enabled=True, env_variables=env_variables) def _log_end(self, build): # TODO fixme config data are not the same as the run part in dynamic steps diff --git a/runbot/tests/common.py b/runbot/tests/common.py index 81cc8e784..101ab72d2 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -227,7 +227,6 @@ def mock_git(repo, cmd, quiet=False, input_data=None, raw=False): self.start_patcher('set_psql_conn_count', 'odoo.addons.runbot.models.host.Host._set_psql_conn_count', None) self.start_patcher('reload_nginx', 'odoo.addons.runbot.models.runbot.Runbot._reload_nginx', None) self.start_patcher('update_commits_infos', 'odoo.addons.runbot.models.batch.Batch._update_commits_infos', None) - self.start_patcher('_local_pg_createdb', 'odoo.addons.runbot.models.build.BuildResult._local_pg_createdb', True) self.start_patcher('getmtime', 'odoo.addons.runbot.common.os.path.getmtime', datetime.datetime.now().timestamp()) self.start_patcher('file_exist', 'odoo.tools.misc.os.path.exists', True) self.start_patcher('_get_py_version', 'odoo.addons.runbot.models.build.BuildResult._get_py_version', 3)