From 928929061fed002c46a36cdf40a2f8536211e55e Mon Sep 17 00:00:00 2001
From: Nicolas Georges <nicolas.georges@creatis.insa-lyon.fr>
Date: Tue, 27 May 2025 12:20:20 +0200
Subject: [PATCH 1/6] Add vipapps.py, an admin tool to manage VIP apps

- Self-contained in vipapps/vipapps.py
- Depends on vipclient, with a minor patch for generic GET/PUT requests
---
 src/vip_client/utils/vip.py |  12 +
 vipapps/README.md           |  10 +
 vipapps/vipapps.py          | 652 ++++++++++++++++++++++++++++++++++++
 3 files changed, 674 insertions(+)
 create mode 100644 vipapps/README.md
 create mode 100644 vipapps/vipapps.py

diff --git a/src/vip_client/utils/vip.py b/src/vip_client/utils/vip.py
index 29ffdd7..cffa411 100644
--- a/src/vip_client/utils/vip.py
+++ b/src/vip_client/utils/vip.py
@@ -308,6 +308,18 @@ def download_parallel(files):
         # Transparent connexion between executor.map() and the caller of download_parallel()
         yield from executor.map(download_thread, files)
 
+def generic_get(endpoint)->list:
+    url = __PREFIX + endpoint
+    rq = SESSION.get(url, headers=__headers)
+    manage_errors(rq)
+    return rq.json()
+
+def generic_put(endpoint,data)->list:
+    url = __PREFIX + endpoint
+    rq = SESSION.put(url, headers=__headers, json=data)
+    manage_errors(rq)
+    return rq.json()
+
 ################################ EXECUTIONS ###################################
 # -----------------------------------------------------------------------------
 def list_executions()->list:
diff --git a/vipapps/README.md b/vipapps/README.md
new file mode 100644
index 0000000..f298890
--- /dev/null
+++ b/vipapps/README.md
@@ -0,0 +1,10 @@
+This is an admin-level tool to manage VIP apps descriptors.
+
+For commands that communicate with a VIP instance, set these two
+environment variables:
+- `export VIP_API_URL=...`  # VIP-portal host URL (without /rest)
+- `export VIP_API_KEY=...`  # Your API key (admin level required)
+
+Then see usage with:
+`python3 ./vipapps/vipapps.py --help`
+or `python3 ./vipapps/vipapps.py <command> --help`
diff --git a/vipapps/vipapps.py b/vipapps/vipapps.py
new file mode 100644
index 0000000..f2e09d0
--- /dev/null
+++ b/vipapps/vipapps.py
@@ -0,0 +1,652 @@
+from vip_client.utils import vip
+import os
+import sys
+import json
+import boutiques
+import contextlib
+import urllib.parse
+import copy
+import re
+import csv
+from pathlib import Path
+import argparse
+
+# global flags
+init_api_done = False
+debug = False
+
+# print message on stderr
+def printerr(*args, **kwargs) -> None:
+    print(*args, file=sys.stderr, **kwargs)
+
+# print message on stderr and exit
+def fatal_error(*args, **kwargs) -> None:
+    printerr(*args, **kwargs)
+    exit(1)
+
+# get VIP server URL
+def get_vip_url() -> str:
+    if not "VIP_API_URL" in os.environ:
+        fatal_error("VIP_API_URL not set")
+    return os.environ["VIP_API_URL"]
+
+# initialize VIP API
+def init_api() -> None:
+    global init_api_done
+    if init_api_done:
+        return
+    # get API key
+    if not "VIP_API_KEY" in os.environ:
+        fatal_error("VIP_API_KEY not set")
+    vip_apikey = os.environ["VIP_API_KEY"]
+    # configure both, in the right order
+    vip.set_vip_url(get_vip_url())
+    vip.setApiKey(vip_apikey)
+    init_api_done = True
+
+# make app object from GET /rest/admin/appVersions response
+def convert_app_version(av):
+    name = av["applicationName"]
+    identifier = name+"/"+av["version"]
+    desc = json.loads(av["descriptor"])
+    return {"name":name,"identifier":identifier,"descriptor":desc,"rawtext":av["descriptor"],"resources":av["resources"],"tags":av["tags"],"is_visible":av["visible"],"settings":av["settings"]}
+
+# get_apps(): get a list of apps and descriptors from a VIP-portal instance
+def get_apps() -> list:
+    init_api()
+    app_versions = vip.generic_get("admin/appVersions")
+    return list(map(convert_app_version, app_versions))
+
+# get_app(): get a single app
+def get_app(identifier) -> object:
+    init_api()
+    try:
+        appver = vip.generic_get("admin/appVersions/"+urllib.parse.quote(identifier))
+    except RuntimeError as e:
+        # XXX this is not a very good way to test whether an app exists or not:
+        # something that explicitly searches for and idea and returns true/false
+        # with 200 OK would be more reliable.
+        # print(e) # Error 8000 from VIP 
+        return None
+    return convert_app_version(appver)
+
+# normalized descriptor filename, assuming name & version are already ok
+def descriptor_filename(appname, appversion):
+    filename = appname + "-" + appversion + ".json"
+    return filename.replace(" ", "_")
+
+# normalized container image name
+def container_image_name(contimg: dict):
+    image = contimg["image"]
+    if ":" not in image:
+        return None
+    # tag is mandatory in this context
+    img_tag = image.split(":")
+    if len(img_tag) != 2:
+        return None
+    items = img_tag[0].split("/")
+    if len(items) < 1:
+        return None
+    name = items[len(items) - 1]
+    tag = img_tag[1]
+    return name + "-" + tag
+
+# a picky checker on the "container-image" section of descriptors:
+# . avoid useless values for "index"
+# . it checks that image names have the form "host.domain/path/repository:tag"
+#   where the host/path part is typically "docker.io/library" for dockerhub,
+#   a URL to some other registry.
+# . a tag must be present and not "latest"
+def check_container_image(filepath: str, contimg: dict) -> None:
+    # container-image.index
+    if "index" in contimg and contimg["index"] != "docker://":
+        printerr("warning: %s: suspicious index '%s'" % (filepath, contimg["index"]))
+    # container-image.image
+    image = contimg["image"]
+    # :tag part
+    if ":" not in image:
+        printerr("warning: %s: image name has no tag" % filepath)
+    else:
+        img_tag = image.split(":")
+        if len(img_tag) != 2:
+            printerr("warning: %s: wrong number of ':'" % filepath)
+        elif img_tag[1] == "latest":
+            printerr("warning: %s: tag 'latest' shouldn't be used" % filepath)
+    # host/path items
+    items = image.split("/")
+    if len(items) < 2 or "." not in items[0]:
+        printerr("warning: %s: image name lacks a host part" % filepath) # "docker.io" on dockerhub
+    elif len(items) < 3:
+        printerr("warning: %s: image name lacks an explicit path" % filepath) # typically "library/" on dockerhub
+
+# load a descriptor from a file, and check its validity
+def load_descriptor(filepath, silent=False) -> dict:
+    try:
+        # just check that we can open the file, to get some cleaner exception
+        # than what boutiques.validate() raises on file not found
+        with open(filepath, "r") as f:
+            pass
+        # bosh validate
+        # https://boutiques.github.io/doc/_validate.html#python-api
+        with open("/dev/null", "w") as f, contextlib.redirect_stdout(f), contextlib.redirect_stderr(f): # silence boutiques output
+            boutiques.validate(str(filepath))
+    # boutiques exception is over-verbose when checking a json file that
+    # doesn't match its schema: catch the relevant exception, and show a
+    # shorter message.
+    # Also note that ValueError is raised by our own checks,
+    # and also a parent of simplejson.errors.JSONDecodeError in boutiques
+    except (ValueError,boutiques.DescriptorValidationError) as e:
+        if type(e) == boutiques.DescriptorValidationError:
+            raise ValueError("bosh validate failed")
+        else:
+            raise e
+    # parse the descriptor again, for our own use
+    file = {"path":filepath, "identifier":"", "descriptor":None, "rawtext":None}
+    with open(filepath, "r") as f:
+        rawtext = f.read()
+        f.seek(0)
+        desc = json.load(f)
+        appname = desc["name"]
+        appversion = desc["tool-version"]
+        # check name and version strings
+        if not re.match(r"^[a-zA-Z0-9_\. +-]+$", appname):
+            raise ValueError("invalid name '%s'" % appname)
+        if not re.match(r"^[a-zA-Z0-9_\.@ +-]+$", appversion):
+            raise ValueError("invalid version '%s'" % appversion)
+        # check container-image (just warnings)
+        if not silent:
+            if not "container-image" in desc:
+                printerr("warning: %s: no container-image" % filepath)
+            else:
+                check_container_image(filepath, desc["container-image"])
+        # store a parsed version of the descriptor in "descriptor",
+        # and the exact original file content in "rawtext"
+        file["identifier"] = appname + "/" + appversion
+        file["descriptor"] = desc
+        file["rawtext"] = rawtext
+    return file
+
+# helper for get_files_*
+def add_file(files, filepath, silent=False):
+    try:
+        file = load_descriptor(filepath, silent=silent)
+        identifier = file["identifier"]
+        if identifier in files and not silent:
+            printerr("ignoring %s: duplicate identifier '%s'"
+                     % (filepath, identifier))
+            return None
+        return file
+    # skip invalid files
+    except ValueError as e:
+        if not silent:
+            printerr("ignoring %s: invalid descriptor (%s)" % (filepath, e))
+            return None
+
+# get an identifier-indexed dict of valid boutiques descriptors in a directory.
+# This uses a flat list of *.json files, no recursive directories walk so far.
+def get_files_from_dir(dirname: str, silent=False) -> dict:
+    local_path = Path(dirname)
+    files = {}
+    for f in local_path.iterdir():
+        if f.is_file() and f.match("*.json"):
+            filepath = local_path.joinpath(f)
+            file = add_file(files, filepath, silent=silent)
+            if file != None:
+                identifier = file["identifier"]
+                files[identifier] = file
+    return files
+
+# same as above, but from a CSV index
+# each app is expected to have three strings: name,version,descriptorPath,
+# resolved from the CSV header
+def get_files_from_index(indexfile: str, silent=False) -> dict:
+    # load csv data
+    csv_header = None
+    csv_rows = []
+    with open(indexfile) as f:
+        rows = csv.reader(f, delimiter=',', quotechar='"')
+        for row in rows:
+            if csv_header == None: # header
+                csv_header = {}
+                for pos,name in enumerate(row):
+                    csv_header[name] = pos
+            else: # data
+                csv_rows.append(row)
+    # check that we have a complete header
+    if not (csv_header
+            and "name" in csv_header
+            and "version" in csv_header
+            and "descriptorPath" in csv_header):
+        fatal_error("%s: missing fields in header" % indexfile)
+    # create files list
+    files = {}
+    for row in csv_rows:
+        filepath = row[csv_header["descriptorPath"]]
+        # descriptor path:
+        if filepath.startswith("/"): # absolute, use as is
+            filepath = Path(filepath)
+        else: # relative, resolve from index
+            filepath = Path(os.path.join(os.path.dirname(indexfile), filepath))
+        # parse descriptor
+        file = add_file(files, filepath, silent=silent)
+        if file != None:
+            # got a valid descriptor file
+            identifier = file["identifier"]
+            desc = file["descriptor"]
+            # check that name and version strings match (fatal)
+            appname = row[csv_header["name"]]
+            appversion = row[csv_header["version"]]
+            if appname != desc["name"] and not silent:
+                printerr("%s: app name '%s' doesn't match descriptor '%s'"
+                         % (filepath, appname, desc["name"]))
+                continue
+            if appversion != desc["tool-version"] and not silent:
+                printerr("%s: app version '%s' doesn't match descriptor '%s'"
+                         % (filepath, appname, desc["tool-version"]))
+                continue
+            # check for normalized descriptor filename (just a warning)
+            normname = descriptor_filename(appname, appversion)
+            if os.path.basename(filepath) != normname:
+                printerr("warning: %s: incorrect descriptor filename, should be '%s'"
+                         % (filepath, normname))
+            # add file to list
+            files[identifier] = file
+    return files
+
+# helper class for the default values of extra fields in apps and appversions
+class AppFields:
+    # XXX TODO: preserve doi, new "origin" field?
+    # XXX somewhat obscure behavior: owner/group/citation are app level,
+    # not appversion, and can't be changed on update (see import_file())
+    # This should either be ajusted to that all fields can be edited (impacting
+    # GET requests), or made more explicit in command-line args.
+
+    # default values for a new app:
+    owner = None
+    groups = []
+    citation = ""
+    resources = []
+    tags = []
+    settings = {}
+    is_visible = True
+    # app is defined on update only, and contains the existing appversion
+    # args is defined everytime, and contains user-provided values
+    def __init__(self, app=None, args=None):
+        # on update, default to keeping existing values
+        if app != None:
+            self.is_visible = app["is_visible"]
+            self.resources = app["resources"]
+            self.tags = app["tags"]
+            self.settings = app["settings"]
+        # override with args values when defined
+        # note an important difference between None and "" here:
+        # . None means the arg was not specified, so its value is unchanged
+        # . "" means 'set to blank'
+        # we only allow a subset of fields in args:
+        # . at app level (creation only): owner and groups
+        # . at appversion level (creation or update): resource
+        if args != None:
+            if args.owner != None:
+                self.owner = None if args.owner == "" else args.owner
+            if args.groups != None:
+                self.groups = [] if args.groups == "" else args.groups.split(",")
+            if args.resources != None:
+                self.resources = [] if args.resources == "" else args.resources.split(",")
+            if args.visible != None:
+                self.is_visible = args.visible
+
+# import an app from a descriptor file to a VIP-portal instance
+# file is assumed already loaded and checked, VIP-portal will re-check anyways
+def import_file(file, fields, is_overwrite=False, dry_run=True):
+    init_api()
+    # create app and appVersion objects for the /rest/admin API
+    appname = file["descriptor"]["name"]
+    version = file["descriptor"]["tool-version"]
+    descriptor = file["rawtext"]
+    app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation}
+    app_url = "admin/applications/" + urllib.parse.quote(appname)
+    appver = {"applicationName":appname,"version":version,"descriptor":descriptor,"visible":fields.is_visible,"resources":fields.resources,"tags":fields.tags,"settings":fields.settings}
+    appver_url = "admin/appVersions/" + urllib.parse.quote(appname) + "/" + urllib.parse.quote(version)
+    msg = ""
+    if is_overwrite:
+        msg += " (overwrite)"
+    if dry_run:
+        msg += " (dry run)"
+    # never change app on update: doing so would be arguable if there are
+    # several appversions per app, and also it avoids having to fetch app
+    # fields on GET (see Appfields())
+    can_put_app = not is_overwrite
+    print("importing app %s %s%s" % (appname, version, msg))
+    if debug:
+        print("descriptor string:", descriptor)
+    if dry_run:
+        if can_put_app:
+            print("PUT %s %s" % (app_url, app))
+        print("PUT %s %s" % (appver_url, appver))
+        return
+    if can_put_app:
+        r = vip.generic_put(app_url, app)
+        if debug:
+            print("app updated:", r)
+    r = vip.generic_put(appver_url, appver)
+    if debug:
+        print("appVersion updated:", r)
+
+# recursive ordering of nested list/dict structures
+# it transforms any dict into a list of 2-tuples to make lists of dicts sortable
+def ordered(obj) -> object:
+    if isinstance(obj, dict):
+        return sorted((k, ordered(v)) for k, v in obj.items())
+    if isinstance(obj, list):
+        return sorted(ordered(x) for x in obj)
+    else:
+        return obj
+
+# delete an array key if present and empty
+def pop_if_empty(desc, field) -> None:
+    if field in desc and type(desc[field]) == list and len(desc[field]) == 0:
+        desc.pop(field)
+
+# descriptor "normalization"
+# When VIP-portal serializes a parsed Descriptor, it adds empty arrays where
+# there were null or missing keys. This is only useful if using an API that
+# doesn't preserve the raw descriptor text.
+def clean_descriptor(desc: dict) -> dict:
+    desc = copy.deepcopy(desc)
+    pop_if_empty(desc, "online-platform-urls")
+    pop_if_empty(desc, "groups")
+    # we don't pop "inputs" as it's mandatory in boutiques
+    pop_if_empty(desc, "output-files")
+    pop_if_empty(desc, "tests")
+    pop_if_empty(desc, "error-codes")
+    pop_if_empty(desc, "environment-variables")
+    if "container-image" in desc:
+        item = desc["container-image"]
+        pop_if_empty(item, "container-opts")
+    for item in desc["inputs"]:
+        pop_if_empty(item, "requires-inputs")
+        pop_if_empty(item, "disables-inputs")
+        pop_if_empty(item, "value-choices")
+    if "output-files" in desc:
+        for item in desc["output-files"]:
+            pop_if_empty(item, "conditional-path-template")
+            pop_if_empty(item, "path-template-stripped-extensions")
+            pop_if_empty(item, "file-template")
+    return desc
+
+# compare two descriptors
+def compare_descriptors(d1, d2) -> bool:
+    # use raw text when available (should always be the case with the admin API)
+    if "rawtext" in d1 and "rawtext" in d2:
+        return d1["rawtext"] == d2["rawtext"]
+    # otherwise, normalize, and ignore keys order for comparison
+    d1 = d1["descriptor"]
+    d2 = d2["descriptor"]
+    return ordered(clean_descriptor(d1))==ordered(clean_descriptor(d2))
+
+# import helpers
+def import_existing_app(app, file, args=None,
+                        show_unchanged=True, is_overwrite=False,
+                        dry_run=True, force_update=False):
+    identifier = app["identifier"]
+    fields = AppFields(app=app, args=args)
+    # XXX here we could also compare non-descriptor fields?
+    if compare_descriptors(app, file) and not force_update:
+        if show_unchanged:
+            print("%s: unchanged" % identifier)
+    elif not is_overwrite:
+        print("%s: changes detected, but overwrite is false" % identifier)
+    else: # import with overwrite
+        print("%s: changes detected, overwriting" % identifier)
+        import_file(file, fields, is_overwrite=True, dry_run=dry_run)
+
+def import_new_app(file, fields, dry_run=True):
+        print("%s: new app" % file["identifier"])
+        import_file(file, fields, is_overwrite=False, dry_run=dry_run)
+
+# sync a list of descriptors with a list of apps (from a VIP instance)
+def perform_sync(args, apps, files):
+    # sort both lists, then do one linear pass on them
+    # we could also use dicts and the sets of their keys.
+    apps.sort(key=lambda app: app["identifier"])
+    files.sort(key=lambda file: file["identifier"])
+    napps = len(apps)
+    nfiles = len(files)
+    i = 0
+    j = 0
+    while i < napps or j < nfiles:
+        app = apps[i] if i < napps else None
+        file = files[j] if j < nfiles else None
+        # if we're not at the end of either list, check if app and file
+        # identifiers match:
+        # . if they don't, process the first one in sort order, and move on
+        # . if they do, compare their descriptors
+        if app != None and file != None:
+            if app["identifier"] < file["identifier"]:
+                file = None
+            elif app["identifier"] > file["identifier"]:
+                app = None
+        if app != None and file != None:
+            # app identifiers match: compare descriptors and import if changed
+            import_existing_app(app, file, args=args,
+                                show_unchanged=args.show_unchanged,
+                                is_overwrite=args.overwrite,
+                                dry_run=args.dry_run,
+                                force_update=args.force_update)
+            i += 1
+            j += 1
+        elif app != None:
+            if args.show_orphans:
+                print("%s: orphan app with no descriptor" % app["identifier"])
+            i += 1
+        elif file != None: # import new app
+            import_new_app(file, AppFields(args=args), dry_run=args.dry_run)
+            j += 1
+
+# helper for list_* commands
+def print_app(label: str, identifier: str, desc: dict,
+              show_descriptor=False, show_imagename=False):
+    print("%s: %s" % (label, identifier))
+    # normalized descriptor filename:
+    # print("  %s" % descriptor_filename(desc["name"], desc["tool-version"]))
+    if show_descriptor:
+        print("  descriptor: %s" % desc)
+    if show_imagename:
+        imagename = None
+        if "container-image" in desc:
+            imagename = container_image_name(desc["container-image"])
+        print("  imagename: %s" % imagename)
+
+# list apps and descriptors on a VIP instance
+def cmd_list_apps(args):
+    apps = get_apps()
+    print("found %d apps on %s:" % (len(apps), get_vip_url()))
+    for app in apps:
+        print_app(app["name"], app["identifier"], app["descriptor"],
+                  show_descriptor=args.show_descriptor,
+                  show_imagename=args.show_imagename)
+
+# list apps from a directory of boutiques descriptors
+def cmd_list_dir(args):
+    descriptors = get_files_from_dir(args.dirname, silent=args.silent)
+    print("found %d valid descriptors in %s:" % (len(descriptors), args.dirname))
+    for identifier in descriptors:
+        file = descriptors[identifier]
+        print_app(file["path"].name, identifier, file["descriptor"],
+                  show_descriptor=args.show_descriptor,
+                  show_imagename=args.show_imagename)
+
+# list apps from a CSV index
+def cmd_list_index(args):
+    descriptors = get_files_from_index(args.filename, silent=args.silent)
+    print("found %d valid descriptors in %s:" % (len(descriptors), args.filename))
+    for identifier in descriptors:
+        file = descriptors[identifier]
+        print_app(file["path"].name, identifier, file["descriptor"],
+                  show_descriptor=args.show_descriptor,
+                  show_imagename=args.show_imagename)
+
+# import a single descriptor file
+def cmd_import_file(args):
+    filepath = args.filename
+    # load and check descriptor
+    file = None
+    try:
+        file = load_descriptor(filepath, silent=args.silent)
+    except ValueError as e:
+        fatal_error("%s is not a valid descriptor: %s" % (filepath, e))
+    # check if app exists
+    appname = file["descriptor"]["name"]
+    version = file["descriptor"]["tool-version"]
+    app = get_app(file["identifier"])
+    if app != None: # app already exists
+        import_existing_app(app, file, args=args, show_unchanged=True,
+                            is_overwrite=args.overwrite,
+                            dry_run=args.dry_run,
+                            force_update=args.force_update)
+    else: # new app
+        import_new_app(file, AppFields(args=args), dry_run=args.dry_run)
+
+# check a single descriptor file
+def cmd_check_file(args):
+    filepath = args.filename
+    try:
+        file = load_descriptor(filepath, silent=args.silent)
+    except ValueError as e:
+        fatal_error("%s is not a valid descriptor: %s" % (filepath, e))
+    print("OK")
+
+# sync apps from a directory of boutiques descriptors to a VIP instance
+def cmd_sync_dir(args):
+    # get apps list from VIP
+    apps = get_apps()
+    # get files list, converting from dict
+    files = get_files_from_dir(args.dirname, silent=args.silent)
+    files = list(files.values())
+    # sync
+    perform_sync(args, apps, files)
+
+# sync apps from a CSV index file to a VIP instance
+def cmd_sync_index(args):
+    # get apps list from VIP
+    apps = get_apps()
+    # get files list
+    files = get_files_from_index(args.filename, silent=args.silent)
+    files = list(files.values())
+    # sync
+    perform_sync(args, apps, files)
+
+def cmd_show_apps(args):
+    print(get_apps())
+
+def cmd_show_files(args):
+    print(get_files_from_dir(args.dirname, silent=args.silent))
+
+def cmd_show_index(args):
+    print(get_files_from_index(args.filename, silent=args.silent))
+
+def add_subcommand(subparsers, name, func, help=None):
+    cmd = subparsers.add_parser(name, help=help)
+    cmd.add_argument("--silent", action="store_true", help="no warnings")
+    cmd.set_defaults(func=func)
+    return cmd
+
+# helper to allow --visible=true/false, with default None
+def parse_bool(val):
+    if isinstance(val, bool):
+        return val
+    if val.lower() in ("true", "1"):
+        return True
+    elif val.lower() in ("false", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("boolean expected")
+
+def add_list_options(cmd):
+    cmd.add_argument("--show-descriptor", action="store_true", help="show parsed descriptor content")
+    cmd.add_argument("--show-imagename", action="store_true", help="show container image name")
+
+def add_import_options(cmd):
+    cmd.add_argument("--dry-run", action="store_true", help="perform no changes, just show what would be done")
+    cmd.add_argument("--overwrite", action="store_true", help="overwrite existing apps")
+    cmd.add_argument("--force-update", action="store_true", help="force update even if descriptor didn't change")
+    cmd.add_argument("--owner", type=str, help="set owner for new apps")
+    cmd.add_argument("--groups", type=str, help="set groups for new apps")
+    cmd.add_argument("--resources", type=str, help="set resources for new or update apps")
+    cmd.add_argument("--visible", type=parse_bool, help="set visibility for new or update apps")
+
+def add_sync_options(cmd):
+    add_import_options(cmd)
+    cmd.add_argument("--show-orphans", action="store_true", help="show apps in VIP-portal with no descriptor in source")
+    cmd.add_argument("--show-unchanged", action="store_true", help="show VIP-portal apps which match their descriptor")
+
+### main
+def main():
+    helpdetail = (
+        "Manage VIP apps descriptors.\n"
+        "For commands that communicate with a VIP instance, set:\n"
+        "export VIP_API_URL=...  # VIP-portal host URL (without /rest)\n"
+        "export VIP_API_KEY=...  # Your API key (admin level required)\n"
+        )
+    parser = argparse.ArgumentParser(prog="vipapps",
+                                     formatter_class=argparse.RawTextHelpFormatter,
+                                     description=helpdetail)
+    subparsers = parser.add_subparsers()
+    # check_file
+    cmd = add_subcommand(subparsers, "check_file", cmd_check_file,
+                         help="verify a descriptor file")
+    cmd.add_argument("filename")
+    # import_file
+    cmd = add_subcommand(subparsers, "import_file", cmd_import_file,
+                         help="import a descriptor file")
+    cmd.add_argument("filename")
+    add_import_options(cmd)
+    # list_apps
+    cmd = add_subcommand(subparsers, "list_apps", cmd_list_apps,
+                         help="list appversions on a VIP instance")
+    add_list_options(cmd)
+    # list_dir
+    cmd = add_subcommand(subparsers, "list_dir", cmd_list_dir,
+                         help="list descriptors in a directory")
+    cmd.add_argument("dirname")
+    add_list_options(cmd)
+    # list_index
+    cmd = add_subcommand(subparsers, "list_index", cmd_list_index,
+                         help="list descriptors in a CSV index")
+    cmd.add_argument("filename")
+    add_list_options(cmd)
+    # sync_dir
+    cmd = add_subcommand(subparsers, "sync_dir", cmd_sync_dir,
+                         help="import descriptors from a directory")
+    cmd.add_argument("dirname")
+    add_sync_options(cmd)
+    # sync_index
+    cmd = add_subcommand(subparsers, "sync_index", cmd_sync_index,
+                         help="import descriptors from a CSV index")
+    cmd.add_argument("filename")
+    add_sync_options(cmd)
+
+    # internal/debug subcommands
+    if debug:
+        cmd = add_subcommand(subparsers, "show_apps", cmd_show_apps,
+                             help=argparse.SUPPRESS)
+        cmd = add_subcommand(subparsers, "show_files", cmd_show_files,
+                             help=argparse.SUPPRESS)
+        cmd.add_argument("dirname")
+        cmd = add_subcommand(subparsers, "show_index", cmd_show_index,
+                             help=argparse.SUPPRESS)
+        cmd.add_argument("filename")
+
+    # parse args
+    argv = sys.argv
+    argv.pop(0)
+    args = parser.parse_args(argv)
+    if not hasattr(args, "func"):
+        parser.print_help()
+        exit(1)
+    args.func(args)
+    exit(0)
+
+### entry point
+if __name__ == "__main__":
+    main()

From 54bb92ddc6ef554b1441be8320f77bdfdbf5c3de Mon Sep 17 00:00:00 2001
From: Nicolas Georges <nicolas.georges@creatis.insa-lyon.fr>
Date: Wed, 28 May 2025 13:33:01 +0200
Subject: [PATCH 2/6] Update vipapps.py

- Add requirements.txt
- Add --public=true/false to define the value of the public field
  when creating an app. As for --groups or --owner, this is currently
  only valid on creation, not on update; and it will use the same value
  for all apps.
- Add --verbose flag and shown PUT requests bodies only when this flag is
  set. Previously, these messages were conditionned by --dry-run.
- Remove --show-unchanged, instead show those with --verbose.
---
 vipapps/requirements.txt |  2 ++
 vipapps/vipapps.py       | 63 ++++++++++++++++++++++------------------
 2 files changed, 36 insertions(+), 29 deletions(-)
 create mode 100644 vipapps/requirements.txt

diff --git a/vipapps/requirements.txt b/vipapps/requirements.txt
new file mode 100644
index 0000000..a0719f3
--- /dev/null
+++ b/vipapps/requirements.txt
@@ -0,0 +1,2 @@
+boutiques==0.5.29
+requests==2.32.3
diff --git a/vipapps/vipapps.py b/vipapps/vipapps.py
index f2e09d0..28a90ad 100644
--- a/vipapps/vipapps.py
+++ b/vipapps/vipapps.py
@@ -256,7 +256,7 @@ def get_files_from_index(indexfile: str, silent=False) -> dict:
 # helper class for the default values of extra fields in apps and appversions
 class AppFields:
     # XXX TODO: preserve doi, new "origin" field?
-    # XXX somewhat obscure behavior: owner/group/citation are app level,
+    # XXX somewhat obscure behavior: owner/group/citation/public are app level,
     # not appversion, and can't be changed on update (see import_file())
     # This should either be ajusted to that all fields can be edited (impacting
     # GET requests), or made more explicit in command-line args.
@@ -265,6 +265,7 @@ class AppFields:
     owner = None
     groups = []
     citation = ""
+    public = False
     resources = []
     tags = []
     settings = {}
@@ -288,6 +289,8 @@ def __init__(self, app=None, args=None):
         if args != None:
             if args.owner != None:
                 self.owner = None if args.owner == "" else args.owner
+            if args.public != None:
+                self.public = args.public
             if args.groups != None:
                 self.groups = [] if args.groups == "" else args.groups.split(",")
             if args.resources != None:
@@ -297,13 +300,13 @@ def __init__(self, app=None, args=None):
 
 # import an app from a descriptor file to a VIP-portal instance
 # file is assumed already loaded and checked, VIP-portal will re-check anyways
-def import_file(file, fields, is_overwrite=False, dry_run=True):
+def import_file(file, fields, is_overwrite=False, dry_run=True, verbose=False):
     init_api()
     # create app and appVersion objects for the /rest/admin API
     appname = file["descriptor"]["name"]
     version = file["descriptor"]["tool-version"]
     descriptor = file["rawtext"]
-    app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation}
+    app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation,"public":fields.public}
     app_url = "admin/applications/" + urllib.parse.quote(appname)
     appver = {"applicationName":appname,"version":version,"descriptor":descriptor,"visible":fields.is_visible,"resources":fields.resources,"tags":fields.tags,"settings":fields.settings}
     appver_url = "admin/appVersions/" + urllib.parse.quote(appname) + "/" + urllib.parse.quote(version)
@@ -317,20 +320,19 @@ def import_file(file, fields, is_overwrite=False, dry_run=True):
     # fields on GET (see Appfields())
     can_put_app = not is_overwrite
     print("importing app %s %s%s" % (appname, version, msg))
-    if debug:
-        print("descriptor string:", descriptor)
-    if dry_run:
-        if can_put_app:
+    if can_put_app:
+        if verbose:
             print("PUT %s %s" % (app_url, app))
+        if not dry_run:
+            r = vip.generic_put(app_url, app)
+            if verbose:
+                print("app updated:", r)
+    if verbose:
         print("PUT %s %s" % (appver_url, appver))
-        return
-    if can_put_app:
-        r = vip.generic_put(app_url, app)
-        if debug:
-            print("app updated:", r)
-    r = vip.generic_put(appver_url, appver)
-    if debug:
-        print("appVersion updated:", r)
+    if not dry_run:
+        r = vip.generic_put(appver_url, appver)
+        if verbose:
+            print("appVersion updated:", r)
 
 # recursive ordering of nested list/dict structures
 # it transforms any dict into a list of 2-tuples to make lists of dicts sortable
@@ -385,24 +387,25 @@ def compare_descriptors(d1, d2) -> bool:
     return ordered(clean_descriptor(d1))==ordered(clean_descriptor(d2))
 
 # import helpers
-def import_existing_app(app, file, args=None,
-                        show_unchanged=True, is_overwrite=False,
-                        dry_run=True, force_update=False):
+def import_existing_app(app, file, args=None, is_overwrite=False,
+                        dry_run=True, verbose=False, force_update=False):
     identifier = app["identifier"]
     fields = AppFields(app=app, args=args)
     # XXX here we could also compare non-descriptor fields?
     if compare_descriptors(app, file) and not force_update:
-        if show_unchanged:
+        if verbose:
             print("%s: unchanged" % identifier)
     elif not is_overwrite:
         print("%s: changes detected, but overwrite is false" % identifier)
     else: # import with overwrite
         print("%s: changes detected, overwriting" % identifier)
-        import_file(file, fields, is_overwrite=True, dry_run=dry_run)
+        import_file(file, fields, is_overwrite=True,
+                    dry_run=dry_run, verbose=verbose)
 
-def import_new_app(file, fields, dry_run=True):
+def import_new_app(file, fields, dry_run=True, verbose=False):
         print("%s: new app" % file["identifier"])
-        import_file(file, fields, is_overwrite=False, dry_run=dry_run)
+        import_file(file, fields, is_overwrite=False,
+                    dry_run=dry_run, verbose=verbose)
 
 # sync a list of descriptors with a list of apps (from a VIP instance)
 def perform_sync(args, apps, files):
@@ -429,9 +432,8 @@ def perform_sync(args, apps, files):
         if app != None and file != None:
             # app identifiers match: compare descriptors and import if changed
             import_existing_app(app, file, args=args,
-                                show_unchanged=args.show_unchanged,
                                 is_overwrite=args.overwrite,
-                                dry_run=args.dry_run,
+                                dry_run=args.dry_run, verbose=args.verbose,
                                 force_update=args.force_update)
             i += 1
             j += 1
@@ -440,7 +442,8 @@ def perform_sync(args, apps, files):
                 print("%s: orphan app with no descriptor" % app["identifier"])
             i += 1
         elif file != None: # import new app
-            import_new_app(file, AppFields(args=args), dry_run=args.dry_run)
+            import_new_app(file, AppFields(args=args),
+                           dry_run=args.dry_run, verbose=args.verbose)
             j += 1
 
 # helper for list_* commands
@@ -500,12 +503,13 @@ def cmd_import_file(args):
     version = file["descriptor"]["tool-version"]
     app = get_app(file["identifier"])
     if app != None: # app already exists
-        import_existing_app(app, file, args=args, show_unchanged=True,
+        import_existing_app(app, file, args=args,
                             is_overwrite=args.overwrite,
-                            dry_run=args.dry_run,
+                            dry_run=args.dry_run, verbose=args.verbose,
                             force_update=args.force_update)
     else: # new app
-        import_new_app(file, AppFields(args=args), dry_run=args.dry_run)
+        import_new_app(file, AppFields(args=args),
+                       dry_run=args.dry_run, verbose=args.verbose)
 
 # check a single descriptor file
 def cmd_check_file(args):
@@ -548,6 +552,7 @@ def cmd_show_index(args):
 def add_subcommand(subparsers, name, func, help=None):
     cmd = subparsers.add_parser(name, help=help)
     cmd.add_argument("--silent", action="store_true", help="no warnings")
+    cmd.add_argument("--verbose", action="store_true", help="show more detail")
     cmd.set_defaults(func=func)
     return cmd
 
@@ -572,13 +577,13 @@ def add_import_options(cmd):
     cmd.add_argument("--force-update", action="store_true", help="force update even if descriptor didn't change")
     cmd.add_argument("--owner", type=str, help="set owner for new apps")
     cmd.add_argument("--groups", type=str, help="set groups for new apps")
+    cmd.add_argument("--public", type=parse_bool, help="set is_public for new apps")
     cmd.add_argument("--resources", type=str, help="set resources for new or update apps")
     cmd.add_argument("--visible", type=parse_bool, help="set visibility for new or update apps")
 
 def add_sync_options(cmd):
     add_import_options(cmd)
     cmd.add_argument("--show-orphans", action="store_true", help="show apps in VIP-portal with no descriptor in source")
-    cmd.add_argument("--show-unchanged", action="store_true", help="show VIP-portal apps which match their descriptor")
 
 ### main
 def main():

From a491a2e9762dbcacce80be1d89060fe59e1ef2d0 Mon Sep 17 00:00:00 2001
From: Nicolas Georges <nicolas.georges@creatis.insa-lyon.fr>
Date: Tue, 3 Jun 2025 17:02:56 +0200
Subject: [PATCH 3/6] Update sync_index

- Ignore rows with a bank descriptor
- Adjust some warning and error messages
---
 vipapps/vipapps.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/vipapps/vipapps.py b/vipapps/vipapps.py
index 28a90ad..35dfa93 100644
--- a/vipapps/vipapps.py
+++ b/vipapps/vipapps.py
@@ -221,7 +221,13 @@ def get_files_from_index(indexfile: str, silent=False) -> dict:
     # create files list
     files = {}
     for row in csv_rows:
+        appname = row[csv_header["name"]]
+        appversion = row[csv_header["version"]]
         filepath = row[csv_header["descriptorPath"]]
+        if filepath == "":
+            if not silent:
+                printerr("%s: no descriptor" % appname)
+            continue
         # descriptor path:
         if filepath.startswith("/"): # absolute, use as is
             filepath = Path(filepath)
@@ -234,19 +240,17 @@ def get_files_from_index(indexfile: str, silent=False) -> dict:
             identifier = file["identifier"]
             desc = file["descriptor"]
             # check that name and version strings match (fatal)
-            appname = row[csv_header["name"]]
-            appversion = row[csv_header["version"]]
-            if appname != desc["name"] and not silent:
-                printerr("%s: app name '%s' doesn't match descriptor '%s'"
+            if appname != desc["name"]:
+                printerr("%s: skipped: app name '%s' doesn't match descriptor '%s'"
                          % (filepath, appname, desc["name"]))
                 continue
-            if appversion != desc["tool-version"] and not silent:
-                printerr("%s: app version '%s' doesn't match descriptor '%s'"
+            if appversion != desc["tool-version"]:
+                printerr("%s: skipped: app version '%s' doesn't match descriptor '%s'"
                          % (filepath, appname, desc["tool-version"]))
                 continue
             # check for normalized descriptor filename (just a warning)
             normname = descriptor_filename(appname, appversion)
-            if os.path.basename(filepath) != normname:
+            if os.path.basename(filepath) != normname and not silent:
                 printerr("warning: %s: incorrect descriptor filename, should be '%s'"
                          % (filepath, normname))
             # add file to list

From 0b2e606ab677daa0b4e7acbc685dc1e0e85d5393 Mon Sep 17 00:00:00 2001
From: Nicolas Georges <nicolas.georges@creatis.insa-lyon.fr>
Date: Thu, 5 Jun 2025 10:14:55 +0200
Subject: [PATCH 4/6] Update vipapp.py

- Add settings and source fields in appversion, with related
  command-line flags on all relevant commands. The source field is
  optional if vip-portal doesn't support it yet
- Preserve doi field
- sync_index: Add support for per-row values for resources, settings,
  and source
---
 vipapps/vipapps.py | 136 +++++++++++++++++++++++++++++++++------------
 1 file changed, 101 insertions(+), 35 deletions(-)

diff --git a/vipapps/vipapps.py b/vipapps/vipapps.py
index 35dfa93..c087e78 100644
--- a/vipapps/vipapps.py
+++ b/vipapps/vipapps.py
@@ -49,7 +49,10 @@ def convert_app_version(av):
     name = av["applicationName"]
     identifier = name+"/"+av["version"]
     desc = json.loads(av["descriptor"])
-    return {"name":name,"identifier":identifier,"descriptor":desc,"rawtext":av["descriptor"],"resources":av["resources"],"tags":av["tags"],"is_visible":av["visible"],"settings":av["settings"]}
+    result = {"name":name,"identifier":identifier,"descriptor":desc,"rawtext":av["descriptor"],"resources":av["resources"],"tags":av["tags"],"is_visible":av["visible"],"settings":av["settings"],"doi":av["doi"]}
+    if "source" in av:
+        result["source"] = av["source"]
+    return result
 
 # get_apps(): get a list of apps and descriptors from a VIP-portal instance
 def get_apps() -> list:
@@ -64,7 +67,7 @@ def get_app(identifier) -> object:
         appver = vip.generic_get("admin/appVersions/"+urllib.parse.quote(identifier))
     except RuntimeError as e:
         # XXX this is not a very good way to test whether an app exists or not:
-        # something that explicitly searches for and idea and returns true/false
+        # something that explicitly searches for an id and returns true/false
         # with 200 OK would be more reliable.
         # print(e) # Error 8000 from VIP 
         return None
@@ -196,7 +199,28 @@ def get_files_from_dir(dirname: str, silent=False) -> dict:
                 files[identifier] = file
     return files
 
-# same as above, but from a CSV index
+# add optional fields from a csv row to a "file" dict
+def csv_add_fields(file, csv_header, row, identifier, silent=False):
+    if "resources" in csv_header:
+        val = row[csv_header["resources"]]
+        try:
+            file["resources"] = parse_strlist(val)
+        except argparse.ArgumentTypeError:
+            if not silent:
+                printerr("warning: %s: ignoring invalid resources '%s'"
+                         % (identifier, val))
+    if "settings" in csv_header:
+        val = row[csv_header["settings"]]
+        try:
+            file["settings"] = parse_map(val)
+        except argparse.ArgumentTypeError as e:
+            if not silent:
+                printerr("warning: %s: ignoring invalid settings '%s'"
+                         % (identifier, val))
+    if "source" in csv_header:
+        file["source"] = row[csv_header["source"]]
+
+# same as get_files_from_dir(), but from a CSV index
 # each app is expected to have three strings: name,version,descriptorPath,
 # resolved from the CSV header
 def get_files_from_index(indexfile: str, silent=False) -> dict:
@@ -246,24 +270,24 @@ def get_files_from_index(indexfile: str, silent=False) -> dict:
                 continue
             if appversion != desc["tool-version"]:
                 printerr("%s: skipped: app version '%s' doesn't match descriptor '%s'"
-                         % (filepath, appname, desc["tool-version"]))
+                         % (filepath, appversion, desc["tool-version"]))
                 continue
             # check for normalized descriptor filename (just a warning)
             normname = descriptor_filename(appname, appversion)
             if os.path.basename(filepath) != normname and not silent:
                 printerr("warning: %s: incorrect descriptor filename, should be '%s'"
                          % (filepath, normname))
+            # add extra fields from csv row
+            csv_add_fields(file, csv_header, row, identifier, silent=silent)
             # add file to list
             files[identifier] = file
     return files
 
 # helper class for the default values of extra fields in apps and appversions
 class AppFields:
-    # XXX TODO: preserve doi, new "origin" field?
-    # XXX somewhat obscure behavior: owner/group/citation/public are app level,
-    # not appversion, and can't be changed on update (see import_file())
-    # This should either be ajusted to that all fields can be edited (impacting
-    # GET requests), or made more explicit in command-line args.
+    # Note a non-obvious behavior: owner/group/citation/public are app-level,
+    # not appversion-level, and can't be changed on update (see import_file())
+    # This should be kept explicit in command-line args.
 
     # default values for a new app:
     owner = None
@@ -274,6 +298,8 @@ class AppFields:
     tags = []
     settings = {}
     is_visible = True
+    doi = None
+    source = ""
     # app is defined on update only, and contains the existing appversion
     # args is defined everytime, and contains user-provided values
     def __init__(self, app=None, args=None):
@@ -283,6 +309,9 @@ def __init__(self, app=None, args=None):
             self.resources = app["resources"]
             self.tags = app["tags"]
             self.settings = app["settings"]
+            self.doi = app["doi"]
+            if "source" in app:
+                self.source = app["source"]
         # override with args values when defined
         # note an important difference between None and "" here:
         # . None means the arg was not specified, so its value is unchanged
@@ -296,11 +325,15 @@ def __init__(self, app=None, args=None):
             if args.public != None:
                 self.public = args.public
             if args.groups != None:
-                self.groups = [] if args.groups == "" else args.groups.split(",")
+                self.groups = args.groups
             if args.resources != None:
-                self.resources = [] if args.resources == "" else args.resources.split(",")
+                self.resources = args.resources
             if args.visible != None:
                 self.is_visible = args.visible
+            if args.settings != None:
+                self.settings = args.settings
+            if args.source != None:
+                self.source = args.source
 
 # import an app from a descriptor file to a VIP-portal instance
 # file is assumed already loaded and checked, VIP-portal will re-check anyways
@@ -312,7 +345,7 @@ def import_file(file, fields, is_overwrite=False, dry_run=True, verbose=False):
     descriptor = file["rawtext"]
     app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation,"public":fields.public}
     app_url = "admin/applications/" + urllib.parse.quote(appname)
-    appver = {"applicationName":appname,"version":version,"descriptor":descriptor,"visible":fields.is_visible,"resources":fields.resources,"tags":fields.tags,"settings":fields.settings}
+    appver = {"applicationName":appname,"version":version,"descriptor":descriptor,"doi":fields.doi,"visible":fields.is_visible,"resources":fields.resources,"tags":fields.tags,"settings":fields.settings,"source":fields.source}
     appver_url = "admin/appVersions/" + urllib.parse.quote(appname) + "/" + urllib.parse.quote(version)
     msg = ""
     if is_overwrite:
@@ -321,7 +354,7 @@ def import_file(file, fields, is_overwrite=False, dry_run=True, verbose=False):
         msg += " (dry run)"
     # never change app on update: doing so would be arguable if there are
     # several appversions per app, and also it avoids having to fetch app
-    # fields on GET (see Appfields())
+    # fields on GET (see AppFields())
     can_put_app = not is_overwrite
     print("importing app %s %s%s" % (appname, version, msg))
     if can_put_app:
@@ -391,10 +424,9 @@ def compare_descriptors(d1, d2) -> bool:
     return ordered(clean_descriptor(d1))==ordered(clean_descriptor(d2))
 
 # import helpers
-def import_existing_app(app, file, args=None, is_overwrite=False,
+def import_existing_app(app, file, fields, is_overwrite=False,
                         dry_run=True, verbose=False, force_update=False):
     identifier = app["identifier"]
-    fields = AppFields(app=app, args=args)
     # XXX here we could also compare non-descriptor fields?
     if compare_descriptors(app, file) and not force_update:
         if verbose:
@@ -433,22 +465,32 @@ def perform_sync(args, apps, files):
                 file = None
             elif app["identifier"] > file["identifier"]:
                 app = None
-        if app != None and file != None:
-            # app identifiers match: compare descriptors and import if changed
-            import_existing_app(app, file, args=args,
-                                is_overwrite=args.overwrite,
-                                dry_run=args.dry_run, verbose=args.verbose,
-                                force_update=args.force_update)
-            i += 1
-            j += 1
+        if file != None:
+            # set field values from global args + per-file overrides
+            fields = AppFields(app=app, args=args)
+            if "resources" in file:
+                fields.resources = file["resources"]
+            if "settings" in file:
+                fields.settings = file["settings"]
+            if "source" in file:
+                fields.source = file["source"]
+            # do the actual sync
+            if app != None:
+                # identifiers match: compare descriptors and import if changed
+                import_existing_app(app, file, fields,
+                                    is_overwrite=args.overwrite,
+                                    dry_run=args.dry_run, verbose=args.verbose,
+                                    force_update=args.force_update)
+                i += 1
+                j += 1
+            else: # app=None,file!=None: import new app
+                import_new_app(file, fields,
+                               dry_run=args.dry_run, verbose=args.verbose)
+                j += 1
         elif app != None:
             if args.show_orphans:
                 print("%s: orphan app with no descriptor" % app["identifier"])
             i += 1
-        elif file != None: # import new app
-            import_new_app(file, AppFields(args=args),
-                           dry_run=args.dry_run, verbose=args.verbose)
-            j += 1
 
 # helper for list_* commands
 def print_app(label: str, identifier: str, desc: dict,
@@ -506,13 +548,14 @@ def cmd_import_file(args):
     appname = file["descriptor"]["name"]
     version = file["descriptor"]["tool-version"]
     app = get_app(file["identifier"])
+    fields = AppFields(app=app, args=args)
     if app != None: # app already exists
-        import_existing_app(app, file, args=args,
+        import_existing_app(app, file, fields,
                             is_overwrite=args.overwrite,
                             dry_run=args.dry_run, verbose=args.verbose,
                             force_update=args.force_update)
     else: # new app
-        import_new_app(file, AppFields(args=args),
+        import_new_app(file, fields,
                        dry_run=args.dry_run, verbose=args.verbose)
 
 # check a single descriptor file
@@ -560,7 +603,7 @@ def add_subcommand(subparsers, name, func, help=None):
     cmd.set_defaults(func=func)
     return cmd
 
-# helper to allow --visible=true/false, with default None
+# parse --flag=true/false, with default None
 def parse_bool(val):
     if isinstance(val, bool):
         return val
@@ -571,6 +614,25 @@ def parse_bool(val):
     else:
         raise argparse.ArgumentTypeError("boolean expected")
 
+# parse --flag=a,b,c and return a list. In practice this can't really fail.
+def parse_strlist(val):
+    if not isinstance(val, str):
+        raise argparse.ArgumentTypeError("string expected")
+    if val == "":
+        return []
+    else:
+        return val.split(",")
+
+# parse --flag='{<json>}', return a dict
+def parse_map(val):
+    try:
+        r = json.loads(val)
+        if not isinstance(r, dict):
+            raise argparse.ArgumentTypeError("json map expected")
+        return r
+    except json.decoder.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError("invalid json",e)
+
 def add_list_options(cmd):
     cmd.add_argument("--show-descriptor", action="store_true", help="show parsed descriptor content")
     cmd.add_argument("--show-imagename", action="store_true", help="show container image name")
@@ -579,11 +641,15 @@ def add_import_options(cmd):
     cmd.add_argument("--dry-run", action="store_true", help="perform no changes, just show what would be done")
     cmd.add_argument("--overwrite", action="store_true", help="overwrite existing apps")
     cmd.add_argument("--force-update", action="store_true", help="force update even if descriptor didn't change")
-    cmd.add_argument("--owner", type=str, help="set owner for new apps")
-    cmd.add_argument("--groups", type=str, help="set groups for new apps")
-    cmd.add_argument("--public", type=parse_bool, help="set is_public for new apps")
-    cmd.add_argument("--resources", type=str, help="set resources for new or update apps")
-    cmd.add_argument("--visible", type=parse_bool, help="set visibility for new or update apps")
+    # app fields (create only)
+    cmd.add_argument("--owner", type=str, help="set owner field (create only)")
+    cmd.add_argument("--groups", type=parse_strlist, help="set groups field (create only)")
+    cmd.add_argument("--public", type=parse_bool, help="set public field (create only)")
+    # appversion fields (create+update)
+    cmd.add_argument("--resources", type=parse_strlist, help="set resources field (create+update)")
+    cmd.add_argument("--visible", type=parse_bool, help="set visible field (create+update)")
+    cmd.add_argument("--settings", type=parse_map, help="set settings field (create+update)")
+    cmd.add_argument("--source", type=str, help="set source field (create+update)")
 
 def add_sync_options(cmd):
     add_import_options(cmd)

From 02c404b1a372dca96cd33c56f3e26da04bf184fe Mon Sep 17 00:00:00 2001
From: Nicolas Georges <nicolas.georges@creatis.insa-lyon.fr>
Date: Tue, 10 Jun 2025 17:17:11 +0200
Subject: [PATCH 5/6] Fix import new appversion with existing parent app

import_file/sync_index/sync_dir commands would previously overwrite
the top-level application record when importing a new app version.
This patch fixes that: a top-level application record can now only be
created, once, when it doesn't exist, and never updated afterwards.
---
 vipapps/vipapps.py | 93 +++++++++++++++++++++++++++++++---------------
 1 file changed, 63 insertions(+), 30 deletions(-)

diff --git a/vipapps/vipapps.py b/vipapps/vipapps.py
index c087e78..6179ad0 100644
--- a/vipapps/vipapps.py
+++ b/vipapps/vipapps.py
@@ -44,8 +44,14 @@ def init_api() -> None:
     vip.setApiKey(vip_apikey)
     init_api_done = True
 
-# make app object from GET /rest/admin/appVersions response
-def convert_app_version(av):
+# make parentapp object from GET /rest/admin/applications response
+def convert_parentapp(app):
+    name = app["name"]
+    result = {"name":name,"owner":app["owner"],"citation":app["citation"],"groups":app["applicationGroups"],"public":app["public"]}
+    return result
+
+# make appversion object from GET /rest/admin/appVersions response
+def convert_appversion(av):
     name = av["applicationName"]
     identifier = name+"/"+av["version"]
     desc = json.loads(av["descriptor"])
@@ -54,14 +60,33 @@ def convert_app_version(av):
         result["source"] = av["source"]
     return result
 
-# get_apps(): get a list of apps and descriptors from a VIP-portal instance
-def get_apps() -> list:
+# get_parentapps(): get a dict of top-level application records
+# from a VIP-portal instance
+def get_parentapps() -> list:
+    init_api()
+    apps = vip.generic_get("admin/applications")
+    appnames = map(lambda app:app["name"], apps)
+    return dict(zip(appnames, list(map(convert_parentapp, apps))))
+
+# get_parentapp(): get a single top-level app
+def get_parentapp(appname) -> object:
+    init_api()
+    try:
+        appver = vip.generic_get("admin/applications/"+urllib.parse.quote(appname))
+    except RuntimeError as e:
+        # XXX see get_appversion
+        return None
+    return convert_parentapp(appver)
+
+# get_appversions(): get a list of appversions and descriptors
+# from a VIP-portal instance
+def get_appversions() -> list:
     init_api()
     app_versions = vip.generic_get("admin/appVersions")
-    return list(map(convert_app_version, app_versions))
+    return list(map(convert_appversion, app_versions))
 
-# get_app(): get a single app
-def get_app(identifier) -> object:
+# get_appversion(): get a single appversion
+def get_appversion(identifier) -> object:
     init_api()
     try:
         appver = vip.generic_get("admin/appVersions/"+urllib.parse.quote(identifier))
@@ -71,7 +96,7 @@ def get_app(identifier) -> object:
         # with 200 OK would be more reliable.
         # print(e) # Error 8000 from VIP 
         return None
-    return convert_app_version(appver)
+    return convert_appversion(appver)
 
 # normalized descriptor filename, assuming name & version are already ok
 def descriptor_filename(appname, appversion):
@@ -290,6 +315,7 @@ class AppFields:
     # This should be kept explicit in command-line args.
 
     # default values for a new app:
+    parent = None
     owner = None
     groups = []
     citation = ""
@@ -302,7 +328,8 @@ class AppFields:
     source = ""
     # app is defined on update only, and contains the existing appversion
     # args is defined everytime, and contains user-provided values
-    def __init__(self, app=None, args=None):
+    def __init__(self, parent=None, app=None, args=None):
+        self.parent = parent
         # on update, default to keeping existing values
         if app != None:
             self.is_visible = app["is_visible"]
@@ -343,27 +370,25 @@ def import_file(file, fields, is_overwrite=False, dry_run=True, verbose=False):
     appname = file["descriptor"]["name"]
     version = file["descriptor"]["tool-version"]
     descriptor = file["rawtext"]
-    app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation,"public":fields.public}
-    app_url = "admin/applications/" + urllib.parse.quote(appname)
-    appver = {"applicationName":appname,"version":version,"descriptor":descriptor,"doi":fields.doi,"visible":fields.is_visible,"resources":fields.resources,"tags":fields.tags,"settings":fields.settings,"source":fields.source}
-    appver_url = "admin/appVersions/" + urllib.parse.quote(appname) + "/" + urllib.parse.quote(version)
     msg = ""
     if is_overwrite:
         msg += " (overwrite)"
     if dry_run:
         msg += " (dry run)"
     # never change app on update: doing so would be arguable if there are
-    # several appversions per app, and also it avoids having to fetch app
-    # fields on GET (see AppFields())
-    can_put_app = not is_overwrite
+    # several appversions per app
     print("importing app %s %s%s" % (appname, version, msg))
-    if can_put_app:
+    if fields.parent == None and is_overwrite == False:
+        app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation,"public":fields.public}
+        app_url = "admin/applications/" + urllib.parse.quote(appname)
         if verbose:
             print("PUT %s %s" % (app_url, app))
         if not dry_run:
             r = vip.generic_put(app_url, app)
             if verbose:
                 print("app updated:", r)
+    appver = {"applicationName":appname,"version":version,"descriptor":descriptor,"doi":fields.doi,"visible":fields.is_visible,"resources":fields.resources,"tags":fields.tags,"settings":fields.settings,"source":fields.source}
+    appver_url = "admin/appVersions/" + urllib.parse.quote(appname) + "/" + urllib.parse.quote(version)
     if verbose:
         print("PUT %s %s" % (appver_url, appver))
     if not dry_run:
@@ -439,12 +464,15 @@ def import_existing_app(app, file, fields, is_overwrite=False,
                     dry_run=dry_run, verbose=verbose)
 
 def import_new_app(file, fields, dry_run=True, verbose=False):
-        print("%s: new app" % file["identifier"])
-        import_file(file, fields, is_overwrite=False,
-                    dry_run=dry_run, verbose=verbose)
+    if fields.parent != None:
+        print("%s: new appversion" % file["identifier"])
+    else:
+        print("%s: new app+appversion" % file["identifier"])
+    import_file(file, fields, is_overwrite=False,
+                dry_run=dry_run, verbose=verbose)
 
 # sync a list of descriptors with a list of apps (from a VIP instance)
-def perform_sync(args, apps, files):
+def perform_sync(args, parents, apps, files):
     # sort both lists, then do one linear pass on them
     # we could also use dicts and the sets of their keys.
     apps.sort(key=lambda app: app["identifier"])
@@ -467,7 +495,9 @@ def perform_sync(args, apps, files):
                 app = None
         if file != None:
             # set field values from global args + per-file overrides
-            fields = AppFields(app=app, args=args)
+            appname = file["descriptor"]["name"]
+            parent = parents[appname] if appname in parents else None
+            fields = AppFields(app=app, args=args, parent=parent)
             if "resources" in file:
                 fields.resources = file["resources"]
             if "settings" in file:
@@ -508,7 +538,7 @@ def print_app(label: str, identifier: str, desc: dict,
 
 # list apps and descriptors on a VIP instance
 def cmd_list_apps(args):
-    apps = get_apps()
+    apps = get_appversions()
     print("found %d apps on %s:" % (len(apps), get_vip_url()))
     for app in apps:
         print_app(app["name"], app["identifier"], app["descriptor"],
@@ -547,8 +577,9 @@ def cmd_import_file(args):
     # check if app exists
     appname = file["descriptor"]["name"]
     version = file["descriptor"]["tool-version"]
-    app = get_app(file["identifier"])
-    fields = AppFields(app=app, args=args)
+    parent = get_parentapp(appname)
+    app = get_appversion(file["identifier"])
+    fields = AppFields(app=app, args=args, parent=parent)
     if app != None: # app already exists
         import_existing_app(app, file, fields,
                             is_overwrite=args.overwrite,
@@ -570,25 +601,27 @@ def cmd_check_file(args):
 # sync apps from a directory of boutiques descriptors to a VIP instance
 def cmd_sync_dir(args):
     # get apps list from VIP
-    apps = get_apps()
+    parents = get_parentapps()
+    apps = get_appversions()
     # get files list, converting from dict
     files = get_files_from_dir(args.dirname, silent=args.silent)
     files = list(files.values())
     # sync
-    perform_sync(args, apps, files)
+    perform_sync(args, parents, apps, files)
 
 # sync apps from a CSV index file to a VIP instance
 def cmd_sync_index(args):
     # get apps list from VIP
-    apps = get_apps()
+    parents = get_parentapps()
+    apps = get_appversions()
     # get files list
     files = get_files_from_index(args.filename, silent=args.silent)
     files = list(files.values())
     # sync
-    perform_sync(args, apps, files)
+    perform_sync(args, parents, apps, files)
 
 def cmd_show_apps(args):
-    print(get_apps())
+    print(get_appversions())
 
 def cmd_show_files(args):
     print(get_files_from_dir(args.dirname, silent=args.silent))

From a06ef7b38fde9b60dddd9ce3e449aba1c0717b28 Mon Sep 17 00:00:00 2001
From: Nicolas Georges <nicolas.georges@creatis.insa-lyon.fr>
Date: Wed, 25 Jun 2025 08:12:58 +0200
Subject: [PATCH 6/6] Update for /rest/admin/applications schema changes

Changes in bean/Application.java,
https://github.com/virtual-imaging-platform/VIP-portal/pull/560:
- Remove isPublic field.
- Rename applicationGroups to groups (+groupNames on read).
  When creating a new app, only the name field is set in group
  objects.
---
 vipapps/vipapps.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/vipapps/vipapps.py b/vipapps/vipapps.py
index 6179ad0..dfdb0c9 100644
--- a/vipapps/vipapps.py
+++ b/vipapps/vipapps.py
@@ -47,7 +47,7 @@ def init_api() -> None:
 # make parentapp object from GET /rest/admin/applications response
 def convert_parentapp(app):
     name = app["name"]
-    result = {"name":name,"owner":app["owner"],"citation":app["citation"],"groups":app["applicationGroups"],"public":app["public"]}
+    result = {"name":name,"owner":app["owner"],"citation":app["citation"],"groups":app["groupsNames"]}
     return result
 
 # make appversion object from GET /rest/admin/appVersions response
@@ -310,7 +310,7 @@ def get_files_from_index(indexfile: str, silent=False) -> dict:
 
 # helper class for the default values of extra fields in apps and appversions
 class AppFields:
-    # Note a non-obvious behavior: owner/group/citation/public are app-level,
+    # Note a non-obvious behavior: owner/group/citation are app-level,
     # not appversion-level, and can't be changed on update (see import_file())
     # This should be kept explicit in command-line args.
 
@@ -319,7 +319,6 @@ class AppFields:
     owner = None
     groups = []
     citation = ""
-    public = False
     resources = []
     tags = []
     settings = {}
@@ -349,8 +348,6 @@ def __init__(self, parent=None, app=None, args=None):
         if args != None:
             if args.owner != None:
                 self.owner = None if args.owner == "" else args.owner
-            if args.public != None:
-                self.public = args.public
             if args.groups != None:
                 self.groups = args.groups
             if args.resources != None:
@@ -379,7 +376,8 @@ def import_file(file, fields, is_overwrite=False, dry_run=True, verbose=False):
     # several appversions per app
     print("importing app %s %s%s" % (appname, version, msg))
     if fields.parent == None and is_overwrite == False:
-        app = {"name":appname,"applicationGroups":fields.groups,"owner":fields.owner,"citation":fields.citation,"public":fields.public}
+        groups = list(map(lambda g:{"name":g}, fields.groups))
+        app = {"name":appname,"groups":groups,"owner":fields.owner,"citation":fields.citation}
         app_url = "admin/applications/" + urllib.parse.quote(appname)
         if verbose:
             print("PUT %s %s" % (app_url, app))
@@ -677,7 +675,6 @@ def add_import_options(cmd):
     # app fields (create only)
     cmd.add_argument("--owner", type=str, help="set owner field (create only)")
     cmd.add_argument("--groups", type=parse_strlist, help="set groups field (create only)")
-    cmd.add_argument("--public", type=parse_bool, help="set public field (create only)")
     # appversion fields (create+update)
     cmd.add_argument("--resources", type=parse_strlist, help="set resources field (create+update)")
     cmd.add_argument("--visible", type=parse_bool, help="set visible field (create+update)")