#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only # # go-dep processor # # Copyright (C) 2022 Bruce Ashfield # Copyright (C) 2023 Chen Qi # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import sys import logging import argparse from collections import OrderedDict import subprocess import textwrap # This switch is used to make this script error out ASAP, mainly for debugging purpose ERROR_OUT_ON_FETCH_AND_CHECKOUT_FAILURE = True logger = logging.getLogger('oe-go-mod-autogen') loggerhandler = logging.StreamHandler() loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) logger.addHandler(loggerhandler) logger.setLevel(logging.INFO) class GoModTool(object): def __init__(self, repo, rev, workdir): self.repo = repo self.rev = rev self.workdir = workdir # Stores the actual module name and its related information # {module: (repo_url, repo_dest_dir, fullsrcrev)} self.modules_repoinfo = {} # {module_name: (url, version, destdir, fullsrcrev)} # # url: place to get the source codes, we only support git repo # version: module version, git tag or git rev # destdir: place to put the fetched source codes # fullsrcrev: full src rev which is the value of SRC_REV # # e.g. # For 'github.com/Masterminds/semver/v3 v3.1.1' in go.mod: # module_name = github.com/Masterminds/semver/v3 # url = https://github.com/Masterminds/semver # version = v3.1.1 # destdir = ${WORKDIR}/${BP}/src/${GO_IMPORT}/vendor/github.com/Masterminds/semver/v3 # fullsrcrev = d387ce7889a157b19ad7694dba39a562051f41b0 self.modules_require = OrderedDict() # {orig_module: (actual_module, actual_version)} self.modules_replace = OrderedDict() # Unhandled modules self.modules_unhandled = OrderedDict() # store subpaths used to form srcpath # {actual_module_name: subpath} self.modules_subpaths = OrderedDict() # modules's actual source paths, record those that are not the same with the module itself self.modules_srcpaths = OrderedDict() # store lines, comment removed self.require_lines = [] self.replace_lines = [] # fetch repo self.fetch_and_checkout_repo(self.repo.split('://')[1], self.repo, self.rev, checkout=True, get_subpath=False) def show_go_mod_info(self): # Print modules_require, modules_replace and modules_unhandled print("modules required:") for m in self.modules_require: url, version, destdir, fullrev = self.modules_require[m] print("%s %s %s %s" % (m, version, url, fullrev)) print("modules replace:") for m in self.modules_replace: actual_module, actual_version = self.modules_replace[m] print("%s => %s %s" % (m, actual_module, actual_version)) print("modules unhandled:") for m in self.modules_unhandled: reason = self.modules_unhandled[m] print("%s unhandled: %s" % (m, reason)) def parse(self): # check if this repo needs autogen repo_url, repo_dest_dir, repo_fullrev = self.modules_repoinfo[self.repo.split('://')[1]] if os.path.isdir(os.path.join(repo_dest_dir, 'vendor')): logger.info("vendor direcotry has already existed for %s, no need to add other repos" % self.repo) return go_mod_file = os.path.join(repo_dest_dir, 'go.mod') if not os.path.exists(go_mod_file): logger.info("go.mod file does not exist for %s, no need to add otehr repos" % self.repo) return self.parse_go_mod(go_mod_file) self.show_go_mod_info() def fetch_and_checkout_repo(self, module_name, repo_url, rev, default_protocol='https://', checkout=False, get_subpath=True): """ Fetch repo_url to /repos/repo_base_name """ protocol = default_protocol if '://' in repo_url: repo_url_final = repo_url else: repo_url_final = default_protocol + repo_url logger.debug("fetch and checkout %s %s" % (repo_url_final, rev)) repos_dir = os.path.join(self.workdir, 'repos') if not os.path.exists(repos_dir): os.makedirs(repos_dir) repo_basename = repo_url.split('/')[-1].split('.git')[0] repo_dest_dir = os.path.join(repos_dir, repo_basename) module_last_name = module_name.split('/')[-1] git_action = "fetch" if os.path.exists(repo_dest_dir): if checkout: # check if current HEAD is rev try: headrev = subprocess.check_output('git rev-list -1 HEAD', shell=True, cwd=repo_dest_dir).decode('utf-8').strip() requiredrev = subprocess.check_output('git rev-list -1 %s 2>/dev/null || git rev-list -1 %s/%s' % (rev, module_last_name, rev), shell=True, cwd=repo_dest_dir).decode('utf-8').strip() if headrev == requiredrev: logger.info("%s has already been fetched and checked out as required, skipping" % repo_url) self.modules_repoinfo[module_name] = (repo_url, repo_dest_dir, requiredrev) return else: logger.info("HEAD of %s is not %s, will do a clean clone" % (repo_dest_dir, requiredrev)) git_action = "clone" except: logger.info("'git rev-list' in %s failed, will do a clean clone" % repo_dest_dir) git_action = "clone" else: # determine if the current repo points to the desired remote repo try: remote_origin_url = subprocess.check_output('git config --get remote.origin.url', shell=True, cwd=repo_dest_dir).decode('utf-8').strip() if remote_origin_url.endswith('.git'): if not repo_url_final.endswith('.git'): remote_origin_url = remote_origin_url[:-4] else: if repo_url_final.endswith('.git'): remote_origin_url = remote_origin_url + '.git' if remote_origin_url != repo_url_final: logger.info("remote.origin.url for %s is not %s, will do a clean clone" % (repo_dest_dir, repo_url_final)) git_action = "clone" except: logger.info("'git config --get remote.origin.url' in %s failed, will do a clean clone" % repo_dest_dir) git_action = "clone" else: # No local repo, clone it. git_action = "clone" if git_action == "clone": logger.info("Removing %s" % repo_dest_dir) subprocess.check_call('rm -rf %s' % repo_dest_dir, shell=True) # clone/fetch repo try: git_cwd = repos_dir if git_action == "clone" else repo_dest_dir logger.info("git %s %s in %s" % (git_action, repo_url_final, git_cwd)) subprocess.check_call('git %s %s >/dev/null 2>&1' % (git_action, repo_url_final), shell=True, cwd=git_cwd) except: logger.warning("Failed to %s %s in %s" % (git_action, repo_url_final, git_cwd)) return def get_requiredrev(get_subpath): import re # check if rev is a revision or a version if len(rev) == 12 and re.match('[0-9a-f]+', rev): rev_is_version = False else: rev_is_version = True # if rev is not a version, 'git rev-list -1 ' should just succeed! if not rev_is_version: try: rev_return = subprocess.check_output('git rev-list -1 %s 2>/dev/null' % rev, shell=True, cwd=repo_dest_dir).decode('utf-8').strip() if get_subpath: cmd = 'git branch -M toremove && git checkout -b check_subpath %s && git branch -D toremove' % rev_return subprocess.check_call(cmd, shell=True, cwd=repo_dest_dir) # try to get the subpath for this module module_name_parts = module_name.split('/') while (len(module_name_parts) > 0): subpath = '/'.join(module_name_parts) dir_to_check = repo_dest_dir + '/' + '/'.join(module_name_parts) if os.path.isdir(dir_to_check): self.modules_subpaths[module_name] = subpath break else: module_name_parts.pop(0) return rev_return except: logger.warning("Revision (%s) not in repo(%s)" % (rev, repo_dest_dir)) return None # the following codes deals with case where rev is a version # determine the longest match tag, in this way, we can get the current srcpath to be used in relocation.inc # we first get the initial tag, which is formed from module_name and rev module_parts = module_name.split('/') if rev.startswith(module_parts[-1] + '.'): tag = '/'.join(module_parts[:-1]) + '/' + rev last_module_part_replaced = True else: tag = '/'.join(module_parts) + '/' + rev last_module_part_replaced = False logger.debug("use %s as the initial tag for %s" % (tag, module_name)) tag_parts = tag.split('/') while(len(tag_parts) > 0): try: rev_return = subprocess.check_output('git rev-list -1 %s 2>/dev/null' % tag, shell=True, cwd=repo_dest_dir).decode('utf-8').strip() if len(tag_parts) > 1: # ensure that the subpath exists if get_subpath: cmd = 'git branch -M toremove && git checkout -b check_subpath %s && git branch -D toremove' % rev_return subprocess.check_call(cmd, shell=True, cwd=repo_dest_dir) # get subpath for the actual_module_name if last_module_part_replaced: subpath = '/'.join(tag_parts[:-1]) + '/' + module_parts[-1] if not os.path.isdir(repo_dest_dir + '/' + subpath): subpath = '/'.join(tag_parts[:-1]) else: subpath = '/'.join(tag_parts[:-1]) if not os.path.isdir(repo_dest_dir + '/' + subpath): logger.warning("subpath (%s) derived from tag matching does not exist in %s" % (subpath, repo_dest_dir)) return None self.modules_subpaths[module_name] = subpath logger.info("modules_subpath[%s] = %s" % (module_name, subpath)) return rev_return except: tag_parts.pop(0) tag = '/'.join(tag_parts) logger.warning("No tag matching %s" % rev) return None requiredrev = get_requiredrev(get_subpath) if requiredrev: logger.info("Got module(%s) requiredrev: %s" % (module_name, requiredrev)) if checkout: subprocess.check_call('git checkout -b gomodautogen %s' % requiredrev, shell=True, cwd=repo_dest_dir) self.modules_repoinfo[module_name] = (repo_url, repo_dest_dir, requiredrev) else: logger.warning("Failed to get requiredrev, repo_url = %s, rev = %s, module_name = %s" % (repo_url, rev, module_name)) def parse_go_mod(self, go_mod_path): """ Parse go.mod file to get the modules info """ # First we get the require and replace lines # The parsing logic assumes the replace lines come *after* the require lines inrequire = False inreplace = False with open(go_mod_path, 'r') as f: lines = f.readlines() for line in lines: if line.startswith('require ('): inrequire = True continue if line.startswith(')'): inrequire = False continue if line.startswith('require ') or inrequire: # we have one line require require_line = line.lstrip('require ').split('//')[0].strip() if require_line: self.require_lines.append(require_line) continue # we can deal with requires and replaces separately because go.mod always writes requires before replaces if line.startswith('replace ('): inreplace = True continue if line.startswith(')'): inreplace = False continue if line.startswith('replace ') or inreplace: replace_line = line.lstrip('replace ').split('//')[0].strip() if replace_line: self.replace_lines.append(replace_line) continue # # parse the require_lines and replace_lines to form self.modules_require and self.modules_replace # logger.debug("Parsing require_lines and replace_lines ...") # A typical replace line is as below: # github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c # It means that the github.com/hashicorp/golang-lru module is replaced by github.com/ktock/golang-lru # with the version 'v0.5.5-0.20211029085301-ec551be6f75c'. # So the destdir is vendor/github.com/hashicorp/golang-lru while the contents are from github.com/ktock/golang-lru for line in self.replace_lines: orig_module, actual = line.split('=>') actual_module, actual_version = actual.split() orig_module = orig_module.strip() actual_module = actual_module.strip() actual_version = actual_version.strip() self.modules_replace[orig_module] = (actual_module, actual_version) # # Typical require lines are as below: # github.com/Masterminds/semver/v3 v3.1.1 # golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 # # We need to first try https://?=go-get=1 to see it contains # line starting with ' # In such case, the self.modules_require has the following contents: # module_name: golang.org/x/crypto # url: https://go.googlesource.com/crypto # version: v0.0.0-20220321153916-2c7772ba3064 # destdir: ${WORKDIR}/${BP}/src/import/vendor.fetch/golang.org/x/crypto # fullsrcrev: 2c7772ba30643b7a2026cbea938420dce7c6384d (git rev-list -1 2c7772ba3064) # # If not, try https://pkg.go.dev/, and find the 'Repository'. # For example, 'wget https://pkg.go.dev/github.com/Masterminds/semver/v3' gives: # github.com/Masterminds/semver # In such case, the self.modules has the following contents: # module_name: github.com/Masterminds/semver/v3 # url: https://github.com/Masterminds/semver # version: v3.1.1 # destdir: ${WORKDIR}/${BP}/src/import/vendor.fetch/github.com/Masterminds/semver/v3 # fullsrcrev: 7bb0c843b53d6ad21a3f619cb22c4b442bb3ef3e (git rev-list -1 v3.1.1) # # As a last resort, if the last component of matches 'v[0-9]+', # remove the last component and try wget https://?go-get=1, # then try using the above matching method. # for line in self.require_lines: module_name, version = line.strip().split() logger.debug("require line: %s" % line) logger.debug("module_name = %s; version = %s" % (module_name, version)) # take the modules_replace into consideration to get the actual version and actual module name # note that the module_name is used in destdir, and the actual_module_name and actual_version # are used to determine the url and fullsrcrev destdir = '${WORKDIR}/${BP}/src/import/vendor.fetch/%s' % module_name actual_module_name = module_name actual_version = version if module_name in self.modules_replace: actual_module_name, actual_version = self.modules_replace[module_name] logger.debug("actual_module_name = %s; actual_version = %s" % (actual_module_name, actual_version)) url, fullsrcrev = self.get_url_srcrev(actual_module_name, actual_version) logger.debug("url = %s; fullsrcrev = %s" % (url, fullsrcrev)) if url and fullsrcrev: self.modules_require[module_name] = (url, version, destdir, fullsrcrev) # form srcpath, actual_module_name/ if actual_module_name in self.modules_subpaths: subpath = self.modules_subpaths[actual_module_name] srcpath = '%s/%s' % (actual_module_name, subpath) self.modules_srcpaths[module_name] = srcpath logger.info("self.modules_srcpaths[%s] = %s" % (module_name, srcpath)) else: self.modules_srcpaths[module_name] = actual_module_name else: logger.warning("get_url_srcrev(%s, %s) failed" % (actual_module_name, actual_version)) if ERROR_OUT_ON_FETCH_AND_CHECKOUT_FAILURE: sys.exit(1) def use_wget_to_get_repo_url(self, wget_content_file, url_cache_file, module_name): """ Use wget to get repo_url for module_name, return None if not found """ try: logger.info("wget -O %s https://%s?=go-get=1" % (wget_content_file, module_name)) subprocess.check_call('wget -O %s https://%s?=go-get=1' % (wget_content_file, module_name), shell=True) with open(wget_content_file, 'r') as f: for line in f.readlines(): if '?=go-get=1, try https://pkg.go/dev/ try: logger.info("wget -O %s https://pkg.go.dev/%s" % (wget_content_file, module_name)) subprocess.check_call("wget -O %s https://pkg.go.dev/%s" % (wget_content_file, module_name), shell=True) repo_url_found = False with open(wget_content_file, 'r') as f: in_repo_section = False for line in f.readlines(): if '>Repository<' in line: in_repo_section = True continue if in_repo_section: newline = line.strip() if newline != '' and not newline.startswith('<'): repo_url = newline repo_url_found = True break if repo_url_found: logger.info("repo url for %s: %s" % (module_name, repo_url)) with open(url_cache_file, 'w') as f: f.write(repo_url) return repo_url else: unhandled_reason = 'cannot determine repo_url for %s' % module_name self.modules_unhandled[module_name] = unhandled_reason return None except: logger.info("wget -O %s https://pkg.go.dev/%s failed" % (wget_content_file, module_name)) return None def get_repo_url_rev(self, module_name, version): """ Return (repo_url, rev) """ import re # First get rev from version v = version.split('+incompatible')[0] version_components = v.split('-') if len(version_components) == 1: rev = v elif len(version_components) == 3: if len(version_components[2]) == 12: rev = version_components[2] else: rev = v else: rev = v # # Get repo_url # We put a cache mechanism here, .repo_url.cache is used to store the repo url fetch before # wget_dir = os.path.join(self.workdir, 'wget-contents') if not os.path.exists(wget_dir): os.makedirs(wget_dir) wget_content_file = os.path.join(wget_dir, module_name.replace('/', '_')) url_cache_file = "%s.repo_url.cache" % wget_content_file if os.path.exists(url_cache_file): with open(url_cache_file, 'r') as f: repo_url = f.readline().strip() return (repo_url, rev) module_name_parts = module_name.split('/') while (len(module_name_parts) > 0): module_name_to_check = '/'.join(module_name_parts) logger.info("module_name_to_check: %s" % module_name_to_check) repo_url = self.use_wget_to_get_repo_url(wget_content_file, url_cache_file, module_name_to_check) if repo_url: return (repo_url, rev) else: if module_name in self.modules_unhandled: return (None, rev) else: module_name_parts.pop(-1) unhandled_reason = 'cannot determine the repo for %s' % module_name self.modules_unhandled[module_name] = unhandled_reason return (None, rev) def get_url_srcrev(self, module_name, version): """ Return url and fullsrcrev according to module_name and version """ repo_url, rev = self.get_repo_url_rev(module_name, version) if not repo_url or not rev: return (None, None) self.fetch_and_checkout_repo(module_name, repo_url, rev) if module_name in self.modules_repoinfo: repo_url, repo_dest_dir, repo_fullrev = self.modules_repoinfo[module_name] # remove the .git suffix to sync repos across modules with different versions and across recipes if repo_url.endswith('.git'): repo_url = repo_url[:-len('.git')] return (repo_url, repo_fullrev) else: unhandled_reason = 'fetch_and_checkout_repo(%s, %s, %s) failed' % (module_name, repo_url, rev) self.modules_unhandled[module_name] = unhandled_reason return (None, None) def gen_src_uri_inc(self): """ Generate src_uri.inc file containing SRC_URIs """ src_uri_inc_file = os.path.join(self.workdir, 'src_uri.inc') # record the after writting SRCREV_, this is to avoid modules having the same basename resulting in same SRCREV_xxx srcrev_name_recorded = [] template = """# %s %s # [1] git ls-remote %s %s SRCREV_%s="%s" SRC_URI += "git://%s;name=%s;protocol=https;nobranch=1;destsuffix=${WORKDIR}/${BP}/src/import/vendor.fetch/%s" """ # We can't simply write SRC_URIs one by one in the order that go.mod specify them. # Because the latter one might clean things up for the former one if the former one is a subpath of the latter one. def take_first_len(elem): return len(elem[0]) src_uri_contents = [] with open(src_uri_inc_file, 'w') as f: for module in self.modules_require: # {module_name: (url, version, destdir, fullsrcrev)} repo_url, version, destdir, fullrev = self.modules_require[module] if module in self.modules_replace: actual_module_name, actual_version = self.modules_replace[module] else: actual_module_name, actual_version = (module, version) if '://' in repo_url: repo_url_noprotocol = repo_url.split('://')[1] else: repo_url_noprotocol = repo_url if not repo_url.startswith('https://'): repo_url = 'https://' + repo_url name = module.split('/')[-1] if name in srcrev_name_recorded: name = '-'.join(module.split('/')[-2:]) src_uri_contents.append((actual_module_name, actual_version, repo_url, fullrev, name, fullrev, repo_url_noprotocol, name, actual_module_name)) srcrev_name_recorded.append(name) # sort the src_uri_contents and then write it src_uri_contents.sort(key=take_first_len) for content in src_uri_contents: f.write(template % content) logger.info("%s generated" % src_uri_inc_file) def gen_relocation_inc(self): """ Generate relocation.inc file """ relocation_inc_file = os.path.join(self.workdir, 'relocation.inc') template = """export sites="%s" do_compile:prepend() { cd ${S}/src/import for s in $sites; do site_dest=$(echo $s | cut -d: -f1) site_source=$(echo $s | cut -d: -f2) force_flag=$(echo $s | cut -d: -f3) mkdir -p vendor.copy/$site_dest if [ -n "$force_flag" ]; then echo "[INFO] $site_dest: force copying .go files" rm -rf vendor.copy/$site_dest rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest else [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; } fi done } """ sites = [] for module in self.modules_require: # :[:force] if module in self.modules_srcpaths: srcpath = self.modules_srcpaths[module] logger.debug("Using %s as srcpath of module (%s)" % (srcpath, module)) else: srcpath = module sites.append("%s:%s:force" % (module, srcpath)) # To avoid the former one being overriden by the latter one when the former one is a subpath of the latter one, sort sites sites.sort(key=len) with open(relocation_inc_file, 'w') as f: sites_str = ' \\\n '.join(sites) f.write(template % sites_str) logger.info("%s generated" % relocation_inc_file) def gen_modules_txt(self): """ Generate modules.txt file """ modules_txt_file = os.path.join(self.workdir, 'modules.txt') with open(modules_txt_file, 'w') as f: for l in self.require_lines: f.write('# %s\n' % l) f.write('## explicit\n') for l in self.replace_lines: f.write('# %s\n' %l) logger.info("%s generated" % modules_txt_file) def sanity_check(self): """ Various anity checks """ sanity_check_ok = True # # Sanity Check 1: # For modules having the same repo, at most one is allowed to not have subpath. # This check operates on self.modules_repoinfo and self.modules_subpaths # repo_modules = {} for module in self.modules_repoinfo: # first form {repo: [module1, module2, ...]} repo_url, repo_dest_dir, fullsrcrev = self.modules_repoinfo[module] if repo_url not in repo_modules: repo_modules[repo_url] = [module] else: repo_modules[repo_url].append(module) for repo in repo_modules: modules = repo_modules[repo] if len(modules) == 1: continue # for modules sharing the same repo, at most one is allowed to not have subpath nosubpath_modules = [] for m in modules: if m not in self.modules_subpaths: nosubpath_modules.append(m) if len(nosubpath_modules) == 0: continue if len(nosubpath_modules) > 1: logger.warning("Multiple modules sharing %s, but they don't have subpath: %s. Please double check." % (repo, nosubpath_modules)) if len(nosubpath_modules) == 1: # do further check, OK if the module is the prefix for other modules sharing the same repo module_to_check = nosubpath_modules[0] for m in modules: if module_to_check == m: continue if not m.startswith('%s/' % module_to_check): logger.warning("%s is sharing repo (%s) with other modules, and it might need a subpath. Please double check: %s and: %s" % (module_to_check, repo, nosubpath_modules,m)) continue # # End of Sanity Check # if not sanity_check_ok: sys.exit(1) return def main(): parser = argparse.ArgumentParser( description="go mod dependency -> SRC_URI procesing", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent('''\ Overview: ========= go-mod-oe is a tool for processing go dependencies to generate dependencies suitable for OE fetcher consumption. In particular, it creates a build structure suitable for '-mod="vendor"' go builds. Once complete all go mod dependencies are in the vendor/ directory, so no golang specific fetching or network access happens during the build. The files src_uri.inc, relocation.inc and modules.txt are generated and suitable for recipe inclusion. A recipe build can then use these files to leverage the git fetcher and related functionality (mirrors, sstate, etc). Note 1: --rev does not have to be a tag, if you want to track the tip of a branch specify the latest git has on that branch, and it will be used. Note 2: This script does not generate an entire recipe, the way the the outputs are used can be modified as required. Note 3: if a go.mod has a bad revision, or needs to be manually updated to fetch fixes: go.mod in the main repository (see the repos/ directory). If go.mod is edited, modules.txt also has to be updated to match the revision information. How to use in a recipe: ======================= There are examples in meta-virtualization of recipes that use this script and stragegy for builds: docker-compose, nerdcli, k3s 1) The recipe should set the master repository SRCREV details, and then include the src_uri.inc file: SRCREV_nerdcli = "e084a2df4a8861eb5f0b0d32df0643ef24b81093" SRC_URI = "git://github.com/containerd/nerdctl.git;name=nerdcli;branch=master;protocol=https" include src_uri.inc This results in the SRC_URI being fully populated with the main repository and all dependencies. 2) The recipe should either copy, or include the relocation.inc file. It sets a variable "sites" that is a list of source locations (where the src_uri.inc fetches) and destination in a vendor directory, it also has a do_compile:prepend() that contains a loop which relocates the fetches into a vendor.copy directory. It is expected to be processed as follows, before compilation starts: # sets the "sites" variable and copies files include relocation.inc The do_compile:prepend, contains the following loop: cd ${S}/src/import # this moves all the fetches into the proper vendor structure # expected for build for s in ${sites}; do site_dest=$(echo $s | cut -d: -f1) site_source=$(echo $s | cut -d: -f2) force_flag=$(echo $s | cut -d: -f3) mkdir -p vendor.copy/$site_dest if [ -n "$force_flag" ]; then echo "[INFO] $site_dest: force copying .go files" rm -rf vendor.copy/$site_dest rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest else [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; } fi done The main compile() function, should set the appropriate GO variables, copy modules.txt and build the appripriate target: # our copied .go files are to be used for the build ln -sf vendor.copy vendor 3) The modules.txt file should be copied into the recipe directory, included on the SRC_URI and copied into place after the relocation has been processed. # patches and config SRC_URI += "file://0001-Makefile-allow-external-specification-of-build-setti.patch \\ file://modules.txt \ " ..... cp ${WORKDIR}/modules.txt vendor/ Example: Updating the K3S recipe ================================ % cd meta-virtualization/recipe-containers/k3s/ # produces src_uri.inc, relocation.inc and modules.txt in the current directory % ../../scripts/oe-go-mod-autogen.py --repo https://github.com/rancher/k3s.git --rev v1.27.5+k3s1 % cp modules.txt k3s/ ... add and commit files. ''')) parser.add_argument("--repo", help = "Repo for the recipe.", required=True) parser.add_argument("--rev", help = "Revision for the recipe.", required=True) parser.add_argument("--module", help = "Go module name. To be used with '--test'") parser.add_argument("--version", help = "Go module version. To be used with '--test'") parser.add_argument("--test", help = "Test to get repo url and fullsrcrev, used together with --module and --version.", action="store_true") parser.add_argument("--workdir", help = "Working directory to hold intermediate results and output.", default=os.getcwd()) parser.add_argument("-d", "--debug", help = "Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_argument("-q", "--quiet", help = "Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") parser.add_argument("-v", action='store_true', dest="verbose", help="verbose") args = parser.parse_args() if args.verbose: args.loglevel = args.verbose args = parser.parse_args() logger.setLevel(args.loglevel) logger.debug("oe-go-mod-autogen.py running for %s:%s in %s" % (args.repo, args.rev, args.workdir)) gomodtool = GoModTool(args.repo, args.rev, args.workdir) if args.test: if not args.module or not args.version: print("Please specify --module and --version") sys.exit(1) url, srcrev = gomodtool.get_url_srcrev(args.module, args.version) print("url = %s, srcrev = %s" % (url, srcrev)) if not url or not srcrev: print("Failed to get url & srcrev for %s:%s" % (args.module, args.version)) else: gomodtool.parse() gomodtool.sanity_check() gomodtool.gen_src_uri_inc() gomodtool.gen_relocation_inc() gomodtool.gen_modules_txt() if __name__ == "__main__": try: ret = main() except Exception as esc: ret = 1 import traceback traceback.print_exc() sys.exit(ret)