Traceback (most recent call last):
  File "/usr/lib/python3.6/site-packages/pungi/scripts/pungi_koji.py", line 637, in cli_main
    main()
  File "/usr/lib/python3.6/site-packages/pungi/scripts/pungi_koji.py", line 337, in main
    latest_link_components=latest_link_components,
  File "/usr/lib/python3.6/site-packages/pungi/scripts/pungi_koji.py", line 495, in run_compose
    essentials_phase.stop()
  File "/usr/lib/python3.6/site-packages/pungi/phases/weaver.py", line 57, in stop
    self.pool.stop()
  File "/usr/lib/python3.6/site-packages/kobo/threads.py", line 120, in stop
    six.reraise(exc_info[0], exc_info[1], exc_info[2])
  File "/usr/lib/python3.6/site-packages/six.py", line 693, in reraise
    raise value
  File "/usr/lib/python3.6/site-packages/kobo/threads.py", line 67, in run
    self.process(item, num)
  File "/usr/lib/python3.6/site-packages/pungi/phases/weaver.py", line 78, in process
    phase.start()
  File "/usr/lib/python3.6/site-packages/pungi/phases/base.py", line 63, in start
    self.run()
  File "/usr/lib/python3.6/site-packages/pungi/phases/gather/__init__.py", line 104, in run
    self.compose, self.pkgset_phase.package_sets, self.pkgset_phase.path_prefix
  File "/usr/lib/python3.6/site-packages/pungi/phases/gather/__init__.py", line 803, in gather_wrapper
    _gather_variants(result, compose, "variant", package_sets)
  File "/usr/lib/python3.6/site-packages/pungi/phases/gather/__init__.py", line 710, in _gather_variants
    variant = compose.all_variants[variant_uid]
KeyError: 'HighAvailability'
Frame _gather_variants in /usr/lib/python3.6/site-packages/pungi/phases/gather/__init__.py at line 710
<CODE>
     703     """Run gathering on all arches of all variants of given type.
     704 
     705     If ``exclude_fulltree`` is set, all source packages from parent variants
     706     will be added to fulltree excludes for the processed variants.
     707     """
     708 
     709     for variant_uid in get_ordered_variant_uids(compose):
-->  710         variant = compose.all_variants[variant_uid]
     711         if variant.type != variant_type:
     712             continue
     713         threads_list = []
     714         que = Queue()
     715         errors = Queue()
     716         for arch in variant.arches:
</CODE>
<LOCALS>
                arch = 'x86_64'
             compose = <pungi.compose.Compose object at 0x7f8d925c4358>
              errors = <queue.Queue object at 0x7f8d803ed550>
    exclude_fulltree = False
   fulltree_excludes = set()
        package_sets = [<pungi.phases.pkgset.common.MaterializedPackageSet object at 0x7f8d81bd2c88>]
             pkg_map = {'rpm': [{'path': '/mnt/koji/packages/Box2D/2.4.1/3.oc9/data/signed/b1ee0065/x86_64/Box2D-2.4.1-3.oc9.x86_64.rpm', 'flags': []}, {'path': '/mnt/koji/packages/Box2D/2.4.1/3.oc9/data/signed/b1ee0065/x86
                 que = <queue.Queue object at 0x7f8d803edc50>
              result = {'x86_64': {'BaseOS': {'rpm': [{'path': '/mnt/koji/packages/less/590/3.oc9/data/signed/b1ee0065/x86_64/less-590-3.oc9.x86_64.rpm', 'flags': ['input', 'prepopulate']}, {'path': '/mnt/koji/packages/zsh/
                   t = <Thread(Thread-64, stopped 140245110617856)>
        threads_list = [<Thread(Thread-63, stopped 140244891723520)>, <Thread(Thread-64, stopped 140245110617856)>]
             variant = Variant(id="AppStream", name="AppStream", type="variant", parent=None)
        variant_type = 'variant'
         variant_uid = 'HighAvailability'
              worker = <function _gather_variants.<locals>.worker at 0x7f8d81c72400>
</LOCALS>
Frame gather_wrapper in /usr/lib/python3.6/site-packages/pungi/phases/gather/__init__.py at line 803
<CODE>
     796                             parent_pkg_map[pkg_type].append(pkg)
     797     return all_included_packages
     798 
     799 
     800 def gather_wrapper(compose, package_sets, path_prefix):
     801     result = {}
     802 
-->  803     _gather_variants(result, compose, "variant", package_sets)
     804     _gather_variants(result, compose, "addon", package_sets, exclude_fulltree=True)
     805     _gather_variants(
     806         result, compose, "layered-product", package_sets, exclude_fulltree=True
     807     )
     808     _gather_variants(result, compose, "optional", package_sets)
     809 
</CODE>
<LOCALS>
             compose = <pungi.compose.Compose object at 0x7f8d925c4358>
        package_sets = [<pungi.phases.pkgset.common.MaterializedPackageSet object at 0x7f8d81bd2c88>]
         path_prefix = '/mnt/koji/'
              result = {'x86_64': {'BaseOS': {'rpm': [{'path': '/mnt/koji/packages/less/590/3.oc9/data/signed/b1ee0065/x86_64/less-590-3.oc9.x86_64.rpm', 'flags': ['input', 'prepopulate']}, {'path': '/mnt/koji/packages/zsh/
</LOCALS>
Frame run in /usr/lib/python3.6/site-packages/pungi/phases/gather/__init__.py at line 104
<CODE>
      97 
      98     def _write_manifest(self):
      99         self.compose.log_info("Writing RPM manifest: %s" % self.manifest_file)
     100         self.manifest.dump(self.manifest_file)
     101 
     102     def run(self):
     103         pkg_map = gather_wrapper(
-->  104             self.compose, self.pkgset_phase.package_sets, self.pkgset_phase.path_prefix
     105         )
     106 
     107         for variant_uid in get_ordered_variant_uids(self.compose):
     108             variant = self.compose.all_variants[variant_uid]
     109             if variant.is_empty:
     110                 continue
</CODE>
<LOCALS>
                self = <pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>
       self._skipped = False
        self.compose = <pungi.compose.Compose object at 0x7f8d925c4358>
       self.finished = False
       self.manifest = <productmd.rpms.Rpms object at 0x7f8d83ed70b8>
  self.manifest_file = '/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/compose/metadata/rpms.json'
            self.msg = '---------- PHASE: GATHER ----------'
           self.name = 'gather'
   self.pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7f8d83ed7080>
  self.used_patterns = None
</LOCALS>
Frame start in /usr/lib/python3.6/site-packages/pungi/phases/base.py at line 63
<CODE>
      56         self._skipped = self.skip()
      57         if self._skipped:
      58             self.compose.log_warning("[SKIP ] %s" % self.msg)
      59             self.finished = True
      60             return
      61         self.compose.log_info("[BEGIN] %s" % self.msg)
      62         self.compose.notifier.send("phase-start", phase_name=self.name)
-->   63         self.run()
      64 
      65     def get_config_block(self, variant, arch=None):
      66         """In config for current phase, find a block corresponding to given
      67         variant and arch. The arch should be given if and only if the config
      68         uses variant/arch mapping.
      69         """
</CODE>
<LOCALS>
                self = <pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>
       self._skipped = False
        self.compose = <pungi.compose.Compose object at 0x7f8d925c4358>
       self.finished = False
       self.manifest = <productmd.rpms.Rpms object at 0x7f8d83ed70b8>
  self.manifest_file = '/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/compose/metadata/rpms.json'
            self.msg = '---------- PHASE: GATHER ----------'
           self.name = 'gather'
   self.pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7f8d83ed7080>
  self.used_patterns = None
</LOCALS>
Frame process in /usr/lib/python3.6/site-packages/pungi/phases/weaver.py at line 78
<CODE>
      71             num,
      72             self.pool.queue_total,
      73             phases_names,
      74         )
      75         self.pool.log_info("[BEGIN] %s" % (msg,))
      76 
      77         for phase in pipeline:
-->   78             phase.start()
      79             phase.stop()
      80 
      81         self.pool.log_info("[DONE ] %s" % (msg,))
</CODE>
<LOCALS>
                item = [<pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>, <pungi.phases.createrepo.CreaterepoPhase object at 0x7f8d83ed74e0>]
                 msg = 'Running pipeline (2/4). Phases: gather, createrepo'
                 num = 2
               phase = <pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>
        phases_names = 'gather, createrepo'
            pipeline = [<pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>, <pungi.phases.createrepo.CreaterepoPhase object at 0x7f8d83ed74e0>]
                self = <PipelineThread(Thread-57, stopped 140244900116224)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140244900116224
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-57'
       self._started = <threading.Event object at 0x7f8d803ed080>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='UTF-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140244900116224
           self.kill = True
           self.name = 'Thread-57'
           self.pool = <kobo.threads.ThreadPool object at 0x7f8d802b28d0>
        self.running = False
</LOCALS>
Frame run in /usr/lib/python3.6/site-packages/kobo/threads.py at line 67
<CODE>
      60 
      61             self.pool.queue_get_lock.acquire()
      62             self.pool.queue_processed += 1
      63             num = self.pool.queue_processed
      64             self.pool.queue_get_lock.release()
      65 
      66             try:
-->   67                 self.process(item, num)
      68             except:
      69                 self.failed = True
      70                 self.pool.exceptions.append(sys.exc_info())
      71                 self.pool.kill()
      72 
      73     def process(self, item, num):
</CODE>
<LOCALS>
                item = [<pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>, <pungi.phases.createrepo.CreaterepoPhase object at 0x7f8d83ed74e0>]
                 num = 2
                self = <PipelineThread(Thread-57, stopped 140244900116224)>
          self._args = <ERROR WHILE CONVERTING VALUE TO STRING>
      self._daemonic = False
         self._ident = 140244900116224
   self._initialized = True
    self._is_stopped = True
        self._kwargs = {}
          self._name = 'Thread-57'
       self._started = <threading.Event object at 0x7f8d803ed080>
        self._stderr = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='UTF-8'>
        self._target = None
   self._tstate_lock = None
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140244900116224
           self.kill = True
           self.name = 'Thread-57'
           self.pool = <kobo.threads.ThreadPool object at 0x7f8d802b28d0>
        self.running = False
</LOCALS>
Frame reraise in /usr/lib/python3.6/site-packages/six.py at line 693
<CODE>
     686 
     687     def reraise(tp, value, tb=None):
     688         try:
     689             if value is None:
     690                 value = tp()
     691             if value.__traceback__ is not tb:
     692                 raise value.with_traceback(tb)
-->  693             raise value
     694         finally:
     695             value = None
     696             tb = None
     697 
     698 else:
     699     def exec_(_code_, _globs_=None, _locs_=None):
</CODE>
<LOCALS>
                  tb = None
                  tp = <class 'KeyError'>
               value = None
</LOCALS>
Frame stop in /usr/lib/python3.6/site-packages/kobo/threads.py at line 120
<CODE>
     113         """
     114         for i in self.threads:
     115             i.running = False
     116         for i in self.threads:
     117             i.join()
     118         if self.exceptions:
     119             exc_info = self.exceptions[0]
-->  120             six.reraise(exc_info[0], exc_info[1], exc_info[2])
     121 
     122     def kill(self):
     123         for i in self.threads:
     124             i.kill = True
     125             i.running = False
     126 
</CODE>
<LOCALS>
            exc_info = <ERROR WHILE CONVERTING VALUE TO STRING>
                   i = <PipelineThread(Thread-60, stopped 140245765166848)>
                self = <kobo.threads.ThreadPool object at 0x7f8d802b28d0>
        self._logger = <Logger pungi (DEBUG)>
     self.exceptions = [(<class 'KeyError'>, KeyError('HighAvailability',), <traceback object at 0x7f8d803f7508>)]
          self.queue = <queue.Queue object at 0x7f8d8029b4e0>
 self.queue_get_lock = <unlocked _thread.lock object at 0x7f8d62cfe440>
self.queue_processed = 4
 self.queue_put_lock = <unlocked _thread.lock object at 0x7f8d62cfe0d0>
    self.queue_total = 4
        self.threads = [<PipelineThread(Thread-57, stopped 140244900116224)>, <PipelineThread(Thread-58, stopped 140245110617856)>, <PipelineThread(Thread-59, stopped 140245224384256)>, <PipelineThread(Thread-60, stopped 14
</LOCALS>
Frame stop in /usr/lib/python3.6/site-packages/pungi/phases/weaver.py at line 57
<CODE>
      50 
      51         self.pool.start()
      52 
      53     def stop(self):
      54         if self.finished:
      55             return
      56         if hasattr(self, "pool"):
-->   57             self.pool.stop()
      58         self.finished = True
      59         self.compose.log_info("[DONE ] %s" % self.msg)
      60 
      61 
      62 class PipelineThread(WorkerThread):
      63     """
</CODE>
<LOCALS>
                self = <pungi.phases.weaver.WeaverPhase object at 0x7f8d636dbf28>
 self._phases_schema = <ERROR WHILE CONVERTING VALUE TO STRING>
        self.compose = <pungi.compose.Compose object at 0x7f8d925c4358>
       self.finished = False
            self.msg = '---------- PHASE: WEAVER ----------'
           self.name = 'weaver'
           self.pool = <kobo.threads.ThreadPool object at 0x7f8d802b28d0>
</LOCALS>
Frame run_compose in /usr/lib/python3.6/site-packages/pungi/scripts/pungi_koji.py at line 495
<CODE>
     488         buildinstall_phase,
     489         (gather_phase, createrepo_phase),
     490         extrafiles_phase,
     491         (ostree_phase, ostree_installer_phase),
     492     )
     493     essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
     494     essentials_phase.start()
-->  495     essentials_phase.stop()
     496 
     497     # write treeinfo before ISOs are created
     498     for variant in compose.get_variants():
     499         for arch in variant.arches + ["src"]:
     500             pungi.metadata.write_tree_info(
     501                 compose, arch, variant, bi=buildinstall_phase
</CODE>
<LOCALS>
  buildinstall_phase = <pungi.phases.buildinstall.BuildinstallPhase object at 0x7f8d83ed72b0>
             compose = <pungi.compose.Compose object at 0x7f8d925c4358>
    config_copy_path = '/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/logs/global/config-copy'
    config_dump_full = '/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/logs/global/config-dump.global.log'
         config_file = '/root/opencloudos/compose/pungi-opencloudos/opencloudos/override.conf'
  create_latest_link = True
     createiso_phase = <pungi.phases.createiso.CreateisoPhase object at 0x7f8d83f169b0>
    createrepo_phase = <pungi.phases.createrepo.CreaterepoPhase object at 0x7f8d83ed74e0>
              errors = []
    essentials_phase = <pungi.phases.weaver.WeaverPhase object at 0x7f8d636dbf28>
   essentials_schema = <ERROR WHILE CONVERTING VALUE TO STRING>
    extra_isos_phase = <pungi.phases.extra_isos.ExtraIsosPhase object at 0x7f8d83ed7be0>
    extrafiles_phase = <pungi.phases.extra_files.ExtraFilesPhase object at 0x7f8d83ed7400>
                   f = <_io.TextIOWrapper name='/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/logs/global/config-dump.global.log' mode='w' encoding='UTF-8'>
        gather_phase = <pungi.phases.gather.GatherPhase object at 0x7f8d83ed7320>
   image_build_phase = <pungi.phases.image_build.ImageBuildPhase object at 0x7f8d83ee45f8>
image_checksum_phase = <pungi.phases.image_checksum.ImageChecksumPhase object at 0x7f8d83ee64a8>
image_container_phase = <pungi.phases.image_container.ImageContainerPhase object at 0x7f8d83ee6128>
          init_phase = <pungi.phases.init.InitPhase object at 0x7f8d83f39ef0>
latest_link_components = -1
  latest_link_status = None
    liveimages_phase = <pungi.phases.live_images.LiveImagesPhase object at 0x7f8d83ed7cc0>
     livemedia_phase = <pungi.phases.livemedia_phase.LiveMediaPhase object at 0x7f8d83ee4278>
          osbs_phase = <pungi.phases.osbs.OSBSPhase object at 0x7f8d83ee4d30>
       osbuild_phase = <pungi.phases.osbuild.OSBuildPhase object at 0x7f8d83ee4978>
ostree_installer_phase = <pungi.phases.ostree_installer.OstreeInstallerPhase object at 0x7f8d83ed76a0>
        ostree_phase = <pungi.phases.ostree.OSTreePhase object at 0x7f8d83ed79e8>
               phase = <pungi.phases.image_container.ImageContainerPhase object at 0x7f8d83ee6128>
        pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7f8d83ed7080>
               pungi = <module 'pungi' from '/usr/lib/python3.6/site-packages/pungi/__init__.py'>
   repoclosure_phase = <pungi.phases.repoclosure.RepoclosurePhase object at 0x7f8d83ee64e0>
          test_phase = <pungi.phases.test.TestPhase object at 0x7f8d83ee6518>
</LOCALS>
Frame main in /usr/lib/python3.6/site-packages/pungi/scripts/pungi_koji.py at line 337
<CODE>
     330     notifier.compose = compose
     331     COMPOSE = compose
     332     try:
     333         run_compose(
     334             compose,
     335             create_latest_link=create_latest_link,
     336             latest_link_status=latest_link_status,
-->  337             latest_link_components=latest_link_components,
     338         )
     339     except UnsignedPackagesError:
     340         # There was an unsigned package somewhere. It is not safe to reuse any
     341         # package set from this compose (since we could leak the unsigned
     342         # package). Let's make sure all reuse files are deleted.
     343         for fp in glob.glob(compose.paths.work.pkgset_reuse_file("*")):
</CODE>
<LOCALS>
             Compose = <class 'pungi.compose.Compose'>
PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'image_checksum', 'image_container', 'init', 'live_images', 'live_media', 'osbs', 'osbuild', 'ostree',
               abort = <function main.<locals>.abort at 0x7f8d848e0048>
             compose = <pungi.compose.Compose object at 0x7f8d925c4358>
         compose_dir = '/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1'
        compose_type = 'production'
                conf = {'RELEASE_NAME': 'OpenCloudOS', 'RELEASE_SHORT': 'OpenCloudOS', 'RELEASE_VERSION': '9.0', 'PKGSET_KOJI_TAG': 'dist-oc9-compose', 'RUNROOT_CHANNEL': 'image', 'RUNROOT_TAG': 'dist-oc9-build', 'RELEASE_V
  create_latest_link = True
              errors = []
       fail_to_start = <function main.<locals>.fail_to_start at 0x7f8d848d7ea0>
               group = <argparse._MutuallyExclusiveGroup object at 0x7f8d92187dd8>
                kobo = <module 'kobo' from '/usr/lib/python3.6/site-packages/kobo/__init__.py'>
latest_link_components = -1
  latest_link_status = None
              logger = <Logger pungi (DEBUG)>
            notifier = <pungi.notifier.PungiNotifier object at 0x7f8d848d2e80>
                opts = Namespace(compose_dir=None, compose_type='production', config='/root/opencloudos/compose/pungi-opencloudos/oc-updates.conf', just_phase=[], koji_event=None, label='RC-9.0', latest_link_components=-1, 
              parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)
           productmd = <module 'productmd' from '/usr/lib/python3.6/site-packages/productmd/__init__.py'>
               pungi = <module 'pungi' from '/usr/lib/python3.6/site-packages/pungi/__init__.py'>
             warning = 'WARNING: Unrecognized config option: RELEASE_VERSION_Z.'
            warnings = ["WARNING: Config option runroot was removed and has no effect; remove it. Please specify 'runroot_method' if you want to enable runroot, otherwise run things locally.", 'WARNING: Config option bootab
</LOCALS>
Frame cli_main in /usr/lib/python3.6/site-packages/pungi/scripts/pungi_koji.py at line 637
<CODE>
     630 
     631 
     632 def cli_main():
     633     signal.signal(signal.SIGINT, sigterm_handler)
     634     signal.signal(signal.SIGTERM, sigterm_handler)
     635 
     636     try:
-->  637         main()
     638     except (Exception, KeyboardInterrupt) as ex:
     639         if COMPOSE:
     640             tb_path = COMPOSE.paths.log.log_file("global", "traceback")
     641             COMPOSE.log_error("Compose run failed: %s" % ex)
     642             COMPOSE.log_error("Extended traceback in: %s" % tb_path)
     643             COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
</CODE>
<LOCALS>
                  ex = KeyError('HighAvailability',)
                   f = <_io.BufferedWriter name='/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/logs/global/traceback.global.log'>
                kobo = <module 'kobo' from '/usr/lib/python3.6/site-packages/kobo/__init__.py'>
             tb_path = '/mnt/koji/compose/9/OpenCloudOS-9.0-updates-20230928.1/logs/global/traceback.global.log'
</LOCALS>