diff options
Diffstat (limited to 'scripts/oe-go-mod-autogen.py')
| -rwxr-xr-x | scripts/oe-go-mod-autogen.py | 800 |
1 files changed, 800 insertions, 0 deletions
diff --git a/scripts/oe-go-mod-autogen.py b/scripts/oe-go-mod-autogen.py new file mode 100755 index 00000000..c44c096a --- /dev/null +++ b/scripts/oe-go-mod-autogen.py @@ -0,0 +1,800 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: GPL-2.0-only +# +# go-dep processor +# +# Copyright (C) 2022 Bruce Ashfield +# Copyright (C) 2023 Chen Qi +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import logging +import argparse +from collections import OrderedDict +import subprocess +import textwrap + +# This switch is used to make this script error out ASAP, mainly for debugging purpose +ERROR_OUT_ON_FETCH_AND_CHECKOUT_FAILURE = True + +logger = logging.getLogger('oe-go-mod-autogen') +loggerhandler = logging.StreamHandler() +loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) +logger.addHandler(loggerhandler) +logger.setLevel(logging.INFO) + +class GoModTool(object): + def __init__(self, repo, rev, workdir): + self.repo = repo + self.rev = rev + self.workdir = workdir + + # Stores the actual module name and its related information + # {module: (repo_url, repo_dest_dir, fullsrcrev)} + self.modules_repoinfo = {} + + # {module_name: (url, version, destdir, fullsrcrev)} + # + # url: place to get the source codes, we only support git repo + # version: module version, git tag or git rev + # destdir: place to put the fetched source codes + # fullsrcrev: full src rev which is the value of SRC_REV + # + # e.g. + # For 'github.com/Masterminds/semver/v3 v3.1.1' in go.mod: + # module_name = github.com/Masterminds/semver/v3 + # url = https://github.com/Masterminds/semver + # version = v3.1.1 + # destdir = ${WORKDIR}/${BP}/src/${GO_IMPORT}/vendor/github.com/Masterminds/semver/v3 + # fullsrcrev = d387ce7889a157b19ad7694dba39a562051f41b0 + self.modules_require = OrderedDict() + + # {orig_module: (actual_module, actual_version)} + self.modules_replace = OrderedDict() + + # Unhandled modules + self.modules_unhandled = OrderedDict() + + # store subpaths used to form srcpath + # {actual_module_name: subpath} + self.modules_subpaths = OrderedDict() + + # modules's actual source paths, record those that are not the same with the module itself + self.modules_srcpaths = OrderedDict() + + # store lines, comment removed + self.require_lines = [] + self.replace_lines = [] + + # fetch repo + self.fetch_and_checkout_repo(self.repo.split('://')[1], self.repo, self.rev, checkout=True, get_subpath=False) + + def show_go_mod_info(self): + # Print modules_require, modules_replace and modules_unhandled + print("modules required:") + for m in self.modules_require: + url, version, destdir, fullrev = self.modules_require[m] + print("%s %s %s %s" % (m, version, url, fullrev)) + + print("modules replace:") + for m in self.modules_replace: + actual_module, actual_version = self.modules_replace[m] + print("%s => %s %s" % (m, actual_module, actual_version)) + + print("modules unhandled:") + for m in self.modules_unhandled: + reason = self.modules_unhandled[m] + print("%s unhandled: %s" % (m, reason)) + + def parse(self): + # check if this repo needs autogen + repo_url, repo_dest_dir, repo_fullrev = self.modules_repoinfo[self.repo.split('://')[1]] + if os.path.isdir(os.path.join(repo_dest_dir, 'vendor')): + logger.info("vendor direcotry has already existed for %s, no need to add other repos" % self.repo) + return + go_mod_file = os.path.join(repo_dest_dir, 'go.mod') + if not os.path.exists(go_mod_file): + logger.info("go.mod file does not exist for %s, no need to add otehr repos" % self.repo) + return + self.parse_go_mod(go_mod_file) + self.show_go_mod_info() + + def fetch_and_checkout_repo(self, module_name, repo_url, rev, default_protocol='https://', checkout=False, get_subpath=True): + """ + Fetch repo_url to <workdir>/repos/repo_base_name + """ + protocol = default_protocol + if '://' in repo_url: + repo_url_final = repo_url + else: + repo_url_final = default_protocol + repo_url + logger.debug("fetch and checkout %s %s" % (repo_url_final, rev)) + repos_dir = os.path.join(self.workdir, 'repos') + if not os.path.exists(repos_dir): + os.makedirs(repos_dir) + repo_basename = repo_url.split('/')[-1].split('.git')[0] + repo_dest_dir = os.path.join(repos_dir, repo_basename) + module_last_name = module_name.split('/')[-1] + git_action = "fetch" + if os.path.exists(repo_dest_dir): + if checkout: + # check if current HEAD is rev + try: + headrev = subprocess.check_output('git rev-list -1 HEAD', shell=True, cwd=repo_dest_dir).decode('utf-8').strip() + requiredrev = subprocess.check_output('git rev-list -1 %s 2>/dev/null || git rev-list -1 %s/%s' % (rev, module_last_name, rev), shell=True, cwd=repo_dest_dir).decode('utf-8').strip() + if headrev == requiredrev: + logger.info("%s has already been fetched and checked out as required, skipping" % repo_url) + self.modules_repoinfo[module_name] = (repo_url, repo_dest_dir, requiredrev) + return + else: + logger.info("HEAD of %s is not %s, will do a clean clone" % (repo_dest_dir, requiredrev)) + git_action = "clone" + except: + logger.info("'git rev-list' in %s failed, will do a clean clone" % repo_dest_dir) + git_action = "clone" + else: + # determine if the current repo points to the desired remote repo + try: + remote_origin_url = subprocess.check_output('git config --get remote.origin.url', shell=True, cwd=repo_dest_dir).decode('utf-8').strip() + if remote_origin_url.endswith('.git'): + if not repo_url_final.endswith('.git'): + remote_origin_url = remote_origin_url[:-4] + else: + if repo_url_final.endswith('.git'): + remote_origin_url = remote_origin_url + '.git' + if remote_origin_url != repo_url_final: + logger.info("remote.origin.url for %s is not %s, will do a clean clone" % (repo_dest_dir, repo_url_final)) + git_action = "clone" + except: + logger.info("'git config --get remote.origin.url' in %s failed, will do a clean clone" % repo_dest_dir) + git_action = "clone" + else: + # No local repo, clone it. + git_action = "clone" + + if git_action == "clone": + logger.info("Removing %s" % repo_dest_dir) + subprocess.check_call('rm -rf %s' % repo_dest_dir, shell=True) + + # clone/fetch repo + try: + git_cwd = repos_dir if git_action == "clone" else repo_dest_dir + logger.info("git %s %s in %s" % (git_action, repo_url_final, git_cwd)) + subprocess.check_call('git %s %s >/dev/null 2>&1' % (git_action, repo_url_final), shell=True, cwd=git_cwd) + except: + logger.warning("Failed to %s %s in %s" % (git_action, repo_url_final, git_cwd)) + return + + def get_requiredrev(get_subpath): + import re + # check if rev is a revision or a version + if len(rev) == 12 and re.match('[0-9a-f]+', rev): + rev_is_version = False + else: + rev_is_version = True + + # if rev is not a version, 'git rev-list -1 <rev>' should just succeed! + if not rev_is_version: + try: + rev_return = subprocess.check_output('git rev-list -1 %s 2>/dev/null' % rev, shell=True, cwd=repo_dest_dir).decode('utf-8').strip() + if get_subpath: + cmd = 'git branch -M toremove && git checkout -b check_subpath %s && git branch -D toremove' % rev_return + subprocess.check_call(cmd, shell=True, cwd=repo_dest_dir) + # try to get the subpath for this module + module_name_parts = module_name.split('/') + while (len(module_name_parts) > 0): + subpath = '/'.join(module_name_parts) + dir_to_check = repo_dest_dir + '/' + '/'.join(module_name_parts) + if os.path.isdir(dir_to_check): + self.modules_subpaths[module_name] = subpath + break + else: + module_name_parts.pop(0) + return rev_return + except: + logger.warning("Revision (%s) not in repo(%s)" % (rev, repo_dest_dir)) + return None + + # the following codes deals with case where rev is a version + # determine the longest match tag, in this way, we can get the current srcpath to be used in relocation.inc + # we first get the initial tag, which is formed from module_name and rev + module_parts = module_name.split('/') + if rev.startswith(module_parts[-1] + '.'): + tag = '/'.join(module_parts[:-1]) + '/' + rev + last_module_part_replaced = True + else: + tag = '/'.join(module_parts) + '/' + rev + last_module_part_replaced = False + logger.debug("use %s as the initial tag for %s" % (tag, module_name)) + tag_parts = tag.split('/') + while(len(tag_parts) > 0): + try: + rev_return = subprocess.check_output('git rev-list -1 %s 2>/dev/null' % tag, shell=True, cwd=repo_dest_dir).decode('utf-8').strip() + if len(tag_parts) > 1: + # ensure that the subpath exists + if get_subpath: + cmd = 'git branch -M toremove && git checkout -b check_subpath %s && git branch -D toremove' % rev_return + subprocess.check_call(cmd, shell=True, cwd=repo_dest_dir) + # get subpath for the actual_module_name + if last_module_part_replaced: + subpath = '/'.join(tag_parts[:-1]) + '/' + module_parts[-1] + if not os.path.isdir(repo_dest_dir + '/' + subpath): + subpath = '/'.join(tag_parts[:-1]) + else: + subpath = '/'.join(tag_parts[:-1]) + if not os.path.isdir(repo_dest_dir + '/' + subpath): + logger.warning("subpath (%s) derived from tag matching does not exist in %s" % (subpath, repo_dest_dir)) + return None + self.modules_subpaths[module_name] = subpath + logger.info("modules_subpath[%s] = %s" % (module_name, subpath)) + return rev_return + except: + tag_parts.pop(0) + tag = '/'.join(tag_parts) + logger.warning("No tag matching %s" % rev) + return None + + requiredrev = get_requiredrev(get_subpath) + if requiredrev: + logger.info("Got module(%s) requiredrev: %s" % (module_name, requiredrev)) + if checkout: + subprocess.check_call('git checkout -b gomodautogen %s' % requiredrev, shell=True, cwd=repo_dest_dir) + self.modules_repoinfo[module_name] = (repo_url, repo_dest_dir, requiredrev) + else: + logger.warning("Failed to get requiredrev, repo_url = %s, rev = %s, module_name = %s" % (repo_url, rev, module_name)) + + def parse_go_mod(self, go_mod_path): + """ + Parse go.mod file to get the modules info + """ + # First we get the require and replace lines + # The parsing logic assumes the replace lines come *after* the require lines + inrequire = False + inreplace = False + with open(go_mod_path, 'r') as f: + lines = f.readlines() + for line in lines: + if line.startswith('require ('): + inrequire = True + continue + if line.startswith(')'): + inrequire = False + continue + if line.startswith('require ') or inrequire: + # we have one line require + require_line = line.lstrip('require ').split('//')[0].strip() + if require_line: + self.require_lines.append(require_line) + continue + # we can deal with requires and replaces separately because go.mod always writes requires before replaces + if line.startswith('replace ('): + inreplace = True + continue + if line.startswith(')'): + inreplace = False + continue + if line.startswith('replace ') or inreplace: + replace_line = line.lstrip('replace ').split('//')[0].strip() + if replace_line: + self.replace_lines.append(replace_line) + continue + # + # parse the require_lines and replace_lines to form self.modules_require and self.modules_replace + # + logger.debug("Parsing require_lines and replace_lines ...") + # A typical replace line is as below: + # github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c + # It means that the github.com/hashicorp/golang-lru module is replaced by github.com/ktock/golang-lru + # with the version 'v0.5.5-0.20211029085301-ec551be6f75c'. + # So the destdir is vendor/github.com/hashicorp/golang-lru while the contents are from github.com/ktock/golang-lru + for line in self.replace_lines: + orig_module, actual = line.split('=>') + actual_module, actual_version = actual.split() + orig_module = orig_module.strip() + actual_module = actual_module.strip() + actual_version = actual_version.strip() + self.modules_replace[orig_module] = (actual_module, actual_version) + # + # Typical require lines are as below: + # github.com/Masterminds/semver/v3 v3.1.1 + # golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 + # + # We need to first try https://<module_name>?=go-get=1 to see it contains + # line starting with '<meta name="go-import" content='. + # + # If so, get root-path vcs repo-url from content. See https://go.dev/ref/mod#vcs-find + # For example, the above 'wget https://golang.org/x/crypto?go-get=1' gives you + # <meta name="go-import" content="golang.org/x/crypto git https://go.googlesource.com/crypto"> + # In such case, the self.modules_require has the following contents: + # module_name: golang.org/x/crypto + # url: https://go.googlesource.com/crypto + # version: v0.0.0-20220321153916-2c7772ba3064 + # destdir: ${WORKDIR}/${BP}/src/import/vendor.fetch/golang.org/x/crypto + # fullsrcrev: 2c7772ba30643b7a2026cbea938420dce7c6384d (git rev-list -1 2c7772ba3064) + # + # If not, try https://pkg.go.dev/<module_name>, and find the 'Repository'. + # For example, 'wget https://pkg.go.dev/github.com/Masterminds/semver/v3' gives: + # github.com/Masterminds/semver + # In such case, the self.modules has the following contents: + # module_name: github.com/Masterminds/semver/v3 + # url: https://github.com/Masterminds/semver + # version: v3.1.1 + # destdir: ${WORKDIR}/${BP}/src/import/vendor.fetch/github.com/Masterminds/semver/v3 + # fullsrcrev: 7bb0c843b53d6ad21a3f619cb22c4b442bb3ef3e (git rev-list -1 v3.1.1) + # + # As a last resort, if the last component of <module_name> matches 'v[0-9]+', + # remove the last component and try wget https://<module_name_with_last_component_removed>?go-get=1, + # then try using the above matching method. + # + for line in self.require_lines: + module_name, version = line.strip().split() + logger.debug("require line: %s" % line) + logger.debug("module_name = %s; version = %s" % (module_name, version)) + # take the modules_replace into consideration to get the actual version and actual module name + # note that the module_name is used in destdir, and the actual_module_name and actual_version + # are used to determine the url and fullsrcrev + destdir = '${WORKDIR}/${BP}/src/import/vendor.fetch/%s' % module_name + actual_module_name = module_name + actual_version = version + if module_name in self.modules_replace: + actual_module_name, actual_version = self.modules_replace[module_name] + logger.debug("actual_module_name = %s; actual_version = %s" % (actual_module_name, actual_version)) + url, fullsrcrev = self.get_url_srcrev(actual_module_name, actual_version) + logger.debug("url = %s; fullsrcrev = %s" % (url, fullsrcrev)) + if url and fullsrcrev: + self.modules_require[module_name] = (url, version, destdir, fullsrcrev) + # form srcpath, actual_module_name/<subpath> + if actual_module_name in self.modules_subpaths: + subpath = self.modules_subpaths[actual_module_name] + srcpath = '%s/%s' % (actual_module_name, subpath) + self.modules_srcpaths[module_name] = srcpath + logger.info("self.modules_srcpaths[%s] = %s" % (module_name, srcpath)) + else: + self.modules_srcpaths[module_name] = actual_module_name + else: + logger.warning("get_url_srcrev(%s, %s) failed" % (actual_module_name, actual_version)) + if ERROR_OUT_ON_FETCH_AND_CHECKOUT_FAILURE: + sys.exit(1) + + def use_wget_to_get_repo_url(self, wget_content_file, url_cache_file, module_name): + """ + Use wget to get repo_url for module_name, return None if not found + """ + try: + logger.info("wget -O %s https://%s?=go-get=1" % (wget_content_file, module_name)) + subprocess.check_call('wget -O %s https://%s?=go-get=1' % (wget_content_file, module_name), shell=True) + with open(wget_content_file, 'r') as f: + for line in f.readlines(): + if '<meta name="go-import" content=' in line: + logger.info("Succeed to find go-import content for %s" % module_name) + logger.debug("The line is %s" % line) + root_path, vcs, repo_url = line.split('content=')[1].split('"')[1].split() + logger.info("%s: %s %s %s" % (module_name, root_path, vcs, repo_url)) + if vcs != 'git': + logger.warning('%s unhandled as its vcs is %s which is not supported by this script.' % (module_name, vcs)) + unhandled_reason = 'vcs %s is not supported by this script' % vcs + self.modules_unhandled[module_name] = unhandled_reason + return None + with open(url_cache_file, 'w') as f: + f.write(repo_url) + return repo_url + except: + logger.info("wget -O %s https://%s?=go-get=1 failed" % (wget_content_file, module_name)) + # if we cannot find repo url from https://<module_name>?=go-get=1, try https://pkg.go/dev/<module_name> + try: + logger.info("wget -O %s https://pkg.go.dev/%s" % (wget_content_file, module_name)) + subprocess.check_call("wget -O %s https://pkg.go.dev/%s" % (wget_content_file, module_name), shell=True) + repo_url_found = False + with open(wget_content_file, 'r') as f: + in_repo_section = False + for line in f.readlines(): + if '>Repository<' in line: + in_repo_section = True + continue + if in_repo_section: + newline = line.strip() + if newline != '' and not newline.startswith('<'): + repo_url = newline + repo_url_found = True + break + if repo_url_found: + logger.info("repo url for %s: %s" % (module_name, repo_url)) + with open(url_cache_file, 'w') as f: + f.write(repo_url) + return repo_url + else: + unhandled_reason = 'cannot determine repo_url for %s' % module_name + self.modules_unhandled[module_name] = unhandled_reason + return None + except: + logger.info("wget -O %s https://pkg.go.dev/%s failed" % (wget_content_file, module_name)) + return None + + + def get_repo_url_rev(self, module_name, version): + """ + Return (repo_url, rev) + """ + import re + # First get rev from version + v = version.split('+incompatible')[0] + version_components = v.split('-') + if len(version_components) == 1: + rev = v + elif len(version_components) == 3: + if len(version_components[2]) == 12: + rev = version_components[2] + else: + rev = v + else: + rev = v + + # + # Get repo_url + # We put a cache mechanism here, <wget_content_file>.repo_url.cache is used to store the repo url fetch before + # + wget_dir = os.path.join(self.workdir, 'wget-contents') + if not os.path.exists(wget_dir): + os.makedirs(wget_dir) + wget_content_file = os.path.join(wget_dir, module_name.replace('/', '_')) + url_cache_file = "%s.repo_url.cache" % wget_content_file + if os.path.exists(url_cache_file): + with open(url_cache_file, 'r') as f: + repo_url = f.readline().strip() + return (repo_url, rev) + module_name_parts = module_name.split('/') + while (len(module_name_parts) > 0): + module_name_to_check = '/'.join(module_name_parts) + logger.info("module_name_to_check: %s" % module_name_to_check) + repo_url = self.use_wget_to_get_repo_url(wget_content_file, url_cache_file, module_name_to_check) + if repo_url: + return (repo_url, rev) + else: + if module_name in self.modules_unhandled: + return (None, rev) + else: + module_name_parts.pop(-1) + + unhandled_reason = 'cannot determine the repo for %s' % module_name + self.modules_unhandled[module_name] = unhandled_reason + return (None, rev) + + def get_url_srcrev(self, module_name, version): + """ + Return url and fullsrcrev according to module_name and version + """ + repo_url, rev = self.get_repo_url_rev(module_name, version) + if not repo_url or not rev: + return (None, None) + self.fetch_and_checkout_repo(module_name, repo_url, rev) + if module_name in self.modules_repoinfo: + repo_url, repo_dest_dir, repo_fullrev = self.modules_repoinfo[module_name] + # remove the .git suffix to sync repos across modules with different versions and across recipes + if repo_url.endswith('.git'): + repo_url = repo_url[:-len('.git')] + return (repo_url, repo_fullrev) + else: + unhandled_reason = 'fetch_and_checkout_repo(%s, %s, %s) failed' % (module_name, repo_url, rev) + self.modules_unhandled[module_name] = unhandled_reason + return (None, None) + + def gen_src_uri_inc(self): + """ + Generate src_uri.inc file containing SRC_URIs + """ + src_uri_inc_file = os.path.join(self.workdir, 'src_uri.inc') + # record the <name> after writting SRCREV_<name>, this is to avoid modules having the same basename resulting in same SRCREV_xxx + srcrev_name_recorded = [] + template = """# %s %s +# [1] git ls-remote %s %s +SRCREV_%s="%s" +SRC_URI += "git://%s;name=%s;protocol=https;nobranch=1;destsuffix=${WORKDIR}/${BP}/src/import/vendor.fetch/%s" + +""" + # We can't simply write SRC_URIs one by one in the order that go.mod specify them. + # Because the latter one might clean things up for the former one if the former one is a subpath of the latter one. + def take_first_len(elem): + return len(elem[0]) + + src_uri_contents = [] + with open(src_uri_inc_file, 'w') as f: + for module in self.modules_require: + # {module_name: (url, version, destdir, fullsrcrev)} + repo_url, version, destdir, fullrev = self.modules_require[module] + if module in self.modules_replace: + actual_module_name, actual_version = self.modules_replace[module] + else: + actual_module_name, actual_version = (module, version) + if '://' in repo_url: + repo_url_noprotocol = repo_url.split('://')[1] + else: + repo_url_noprotocol = repo_url + if not repo_url.startswith('https://'): + repo_url = 'https://' + repo_url + name = module.split('/')[-1] + if name in srcrev_name_recorded: + name = '-'.join(module.split('/')[-2:]) + src_uri_contents.append((actual_module_name, actual_version, repo_url, fullrev, name, fullrev, repo_url_noprotocol, name, actual_module_name)) + srcrev_name_recorded.append(name) + # sort the src_uri_contents and then write it + src_uri_contents.sort(key=take_first_len) + for content in src_uri_contents: + f.write(template % content) + logger.info("%s generated" % src_uri_inc_file) + + def gen_relocation_inc(self): + """ + Generate relocation.inc file + """ + relocation_inc_file = os.path.join(self.workdir, 'relocation.inc') + template = """export sites="%s" + +do_compile:prepend() { + cd ${S}/src/import + for s in $sites; do + site_dest=$(echo $s | cut -d: -f1) + site_source=$(echo $s | cut -d: -f2) + force_flag=$(echo $s | cut -d: -f3) + mkdir -p vendor.copy/$site_dest + if [ -n "$force_flag" ]; then + echo "[INFO] $site_dest: force copying .go files" + rm -rf vendor.copy/$site_dest + rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest + else + [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; } + fi + done +} +""" + sites = [] + for module in self.modules_require: + # <dest>:<source>[:force] + if module in self.modules_srcpaths: + srcpath = self.modules_srcpaths[module] + logger.debug("Using %s as srcpath of module (%s)" % (srcpath, module)) + else: + srcpath = module + sites.append("%s:%s:force" % (module, srcpath)) + # To avoid the former one being overriden by the latter one when the former one is a subpath of the latter one, sort sites + sites.sort(key=len) + with open(relocation_inc_file, 'w') as f: + sites_str = ' \\\n '.join(sites) + f.write(template % sites_str) + logger.info("%s generated" % relocation_inc_file) + + def gen_modules_txt(self): + """ + Generate modules.txt file + """ + modules_txt_file = os.path.join(self.workdir, 'modules.txt') + with open(modules_txt_file, 'w') as f: + for l in self.require_lines: + f.write('# %s\n' % l) + f.write('## explicit\n') + for l in self.replace_lines: + f.write('# %s\n' %l) + logger.info("%s generated" % modules_txt_file) + + def sanity_check(self): + """ + Various anity checks + """ + sanity_check_ok = True + # + # Sanity Check 1: + # For modules having the same repo, at most one is allowed to not have subpath. + # This check operates on self.modules_repoinfo and self.modules_subpaths + # + repo_modules = {} + for module in self.modules_repoinfo: + # first form {repo: [module1, module2, ...]} + repo_url, repo_dest_dir, fullsrcrev = self.modules_repoinfo[module] + if repo_url not in repo_modules: + repo_modules[repo_url] = [module] + else: + repo_modules[repo_url].append(module) + for repo in repo_modules: + modules = repo_modules[repo] + if len(modules) == 1: + continue + # for modules sharing the same repo, at most one is allowed to not have subpath + nosubpath_modules = [] + for m in modules: + if m not in self.modules_subpaths: + nosubpath_modules.append(m) + if len(nosubpath_modules) == 0: + continue + if len(nosubpath_modules) > 1: + logger.warning("Multiple modules sharing %s, but they don't have subpath: %s. Please double check." % (repo, nosubpath_modules)) + if len(nosubpath_modules) == 1: + # do further check, OK if the module is the prefix for other modules sharing the same repo + module_to_check = nosubpath_modules[0] + for m in modules: + if module_to_check == m: + continue + if not m.startswith('%s/' % module_to_check): + logger.warning("%s is sharing repo (%s) with other modules, and it might need a subpath. Please double check: %s and: %s" % (module_to_check, repo, nosubpath_modules,m)) + continue + + # + # End of Sanity Check + # + if not sanity_check_ok: + sys.exit(1) + return + +def main(): + parser = argparse.ArgumentParser( + description="go mod dependency -> SRC_URI procesing", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent('''\ + + Overview: + ========= + + go-mod-oe is a tool for processing go dependencies to generate + dependencies suitable for OE fetcher consumption. + + In particular, it creates a build structure suitable for + '-mod="vendor"' go builds. Once complete all go mod dependencies + are in the vendor/ directory, so no golang specific fetching or + network access happens during the build. + + The files src_uri.inc, relocation.inc and modules.txt are generated + and suitable for recipe inclusion. + + A recipe build can then use these files to leverage the git fetcher + and related functionality (mirrors, sstate, etc). + + Note 1: --rev does not have to be a tag, if you want to track the tip of + a branch specify the latest git has on that branch, and it will + be used. + + Note 2: This script does not generate an entire recipe, the way the + the outputs are used can be modified as required. + + Note 3: if a go.mod has a bad revision, or needs to be manually updated + to fetch fixes: go.mod in the main repository (see the repos/ + directory). If go.mod is edited, modules.txt also has to be + updated to match the revision information. + + How to use in a recipe: + ======================= + + There are examples in meta-virtualization of recipes that use this + script and stragegy for builds: docker-compose, nerdcli, k3s + + 1) The recipe should set the master repository SRCREV details, and then include + the src_uri.inc file: + + SRCREV_nerdcli = "e084a2df4a8861eb5f0b0d32df0643ef24b81093" + SRC_URI = "git://github.com/containerd/nerdctl.git;name=nerdcli;branch=master;protocol=https" + + include src_uri.inc + + This results in the SRC_URI being fully populated with the main + repository and all dependencies. + + 2) The recipe should either copy, or include the relocation.inc file. It sets + a variable "sites" that is a list of source locations (where the src_uri.inc + fetches) and destination in a vendor directory, it also has a do_compile:prepend() + that contains a loop which relocates the fetches into a vendor.copy directory. + + It is expected to be processed as follows, before compilation starts: + + # sets the "sites" variable and copies files + include relocation.inc + + The do_compile:prepend, contains the following loop: + + cd ${S}/src/import + # this moves all the fetches into the proper vendor structure + # expected for build + for s in ${sites}; do + site_dest=$(echo $s | cut -d: -f1) + site_source=$(echo $s | cut -d: -f2) + force_flag=$(echo $s | cut -d: -f3) + mkdir -p vendor.copy/$site_dest + if [ -n "$force_flag" ]; then + echo "[INFO] $site_dest: force copying .go files" + rm -rf vendor.copy/$site_dest + rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest + else + [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; } + fi + done + + The main compile() function, should set the appropriate GO variables, + copy modules.txt and build the appripriate target: + + # our copied .go files are to be used for the build + ln -sf vendor.copy vendor + + 3) The modules.txt file should be copied into the recipe directory, included + on the SRC_URI and copied into place after the relocation has been + processed. + + # patches and config + SRC_URI += "file://0001-Makefile-allow-external-specification-of-build-setti.patch \\ + file://modules.txt \ + " + + ..... + + cp ${WORKDIR}/modules.txt vendor/ + + Example: Updating the K3S recipe + ================================ + + % cd meta-virtualization/recipe-containers/k3s/ + # produces src_uri.inc, relocation.inc and modules.txt in the current directory + % ../../scripts/oe-go-mod-autogen.py --repo https://github.com/rancher/k3s.git --rev v1.27.5+k3s1 + + % cp modules.txt k3s/ + + ... add and commit files. + + + ''')) + parser.add_argument("--repo", help = "Repo for the recipe.", required=True) + parser.add_argument("--rev", help = "Revision for the recipe.", required=True) + parser.add_argument("--module", help = "Go module name. To be used with '--test'") + parser.add_argument("--version", help = "Go module version. To be used with '--test'") + parser.add_argument("--test", help = "Test to get repo url and fullsrcrev, used together with --module and --version.", action="store_true") + parser.add_argument("--workdir", help = "Working directory to hold intermediate results and output.", default=os.getcwd()) + parser.add_argument("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) + parser.add_argument("-q", "--quiet", + help = "Hide all output except error messages", + action="store_const", const=logging.ERROR, dest="loglevel") + parser.add_argument("-v", action='store_true', dest="verbose", + help="verbose") + + args = parser.parse_args() + + if args.verbose: + args.loglevel = args.verbose + args = parser.parse_args() + + logger.setLevel(args.loglevel) + logger.debug("oe-go-mod-autogen.py running for %s:%s in %s" % (args.repo, args.rev, args.workdir)) + gomodtool = GoModTool(args.repo, args.rev, args.workdir) + if args.test: + if not args.module or not args.version: + print("Please specify --module and --version") + sys.exit(1) + url, srcrev = gomodtool.get_url_srcrev(args.module, args.version) + print("url = %s, srcrev = %s" % (url, srcrev)) + if not url or not srcrev: + print("Failed to get url & srcrev for %s:%s" % (args.module, args.version)) + else: + gomodtool.parse() + gomodtool.sanity_check() + gomodtool.gen_src_uri_inc() + gomodtool.gen_relocation_inc() + gomodtool.gen_modules_txt() + + +if __name__ == "__main__": + try: + ret = main() + except Exception as esc: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) |