From 91f49c0700dde13e16f42c745584a0bef60c6fe2 Mon Sep 17 00:00:00 2001 From: Anthony Fok Date: Sun, 30 Sep 2018 16:28:11 -0600 Subject: [PATCH] snap: Add custom x-nodejs plugin to support ppc64el and s390x --- snap/plugins/x-nodejs.yaml | 8 + snap/plugins/x_nodejs.py | 332 +++++++++++++++++++++++++++++++++++++ snap/snapcraft.yaml | 2 +- 3 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 snap/plugins/x-nodejs.yaml create mode 100644 snap/plugins/x_nodejs.py diff --git a/snap/plugins/x-nodejs.yaml b/snap/plugins/x-nodejs.yaml new file mode 100644 index 000000000..60b465459 --- /dev/null +++ b/snap/plugins/x-nodejs.yaml @@ -0,0 +1,8 @@ +options: + source: + required: true + source-type: + source-tag: + source-branch: + nodejs-target: + required: true diff --git a/snap/plugins/x_nodejs.py b/snap/plugins/x_nodejs.py new file mode 100644 index 000000000..848cac596 --- /dev/null +++ b/snap/plugins/x_nodejs.py @@ -0,0 +1,332 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Modified by Anthony Fok on 2018-10-01 to add support for ppc64el and s390x +# +# Copyright (C) 2015-2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The nodejs plugin is useful for node/npm based parts. + +The plugin uses node to install dependencies from `package.json`. It +also sets up binaries defined in `package.json` into the `PATH`. + +This plugin uses the common plugin keywords as well as those for "sources". +For more information check the 'plugins' topic for the former and the +'sources' topic for the latter. + +Additionally, this plugin uses the following plugin-specific keywords: + + - node-packages: + (list) + A list of dependencies to fetch using npm. + - node-engine: + (string) + The version of nodejs you want the snap to run on. + - npm-run: + (list) + A list of targets to `npm run`. + These targets will be run in order, after `npm install` + - npm-flags: + (list) + A list of flags for npm. + - node-package-manager + (string; default: npm) + The language package manager to use to drive installation + of node packages. Can be either `npm` (default) or `yarn`. +""" + +import collections +import contextlib +import json +import logging +import os +import shutil +import subprocess +import sys + +import snapcraft +from snapcraft import sources +from snapcraft.file_utils import link_or_copy_tree +from snapcraft.internal import errors + +logger = logging.getLogger(__name__) + +_NODEJS_BASE = "node-v{version}-linux-{arch}" +_NODEJS_VERSION = "8.12.0" +_NODEJS_TMPL = "https://nodejs.org/dist/v{version}/{base}.tar.gz" +_NODEJS_ARCHES = {"i386": "x86", "amd64": "x64", "armhf": "armv7l", "arm64": "arm64", "ppc64el": "ppc64le", "s390x": "s390x"} +_YARN_URL = "https://yarnpkg.com/latest.tar.gz" + + +class NodePlugin(snapcraft.BasePlugin): + @classmethod + def schema(cls): + schema = super().schema() + + schema["properties"]["node-packages"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["node-engine"] = { + "type": "string", + "default": _NODEJS_VERSION, + } + schema["properties"]["node-package-manager"] = { + "type": "string", + "default": "npm", + "enum": ["npm", "yarn"], + } + schema["properties"]["npm-run"] = { + "type": "array", + "minitems": 1, + "uniqueItems": False, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["npm-flags"] = { + "type": "array", + "minitems": 1, + "uniqueItems": False, + "items": {"type": "string"}, + "default": [], + } + + if "required" in schema: + del schema["required"] + + return schema + + @classmethod + def get_build_properties(cls): + # Inform Snapcraft of the properties associated with building. If these + # change in the YAML Snapcraft will consider the build step dirty. + return ["node-packages", "npm-run", "npm-flags"] + + @classmethod + def get_pull_properties(cls): + # Inform Snapcraft of the properties associated with pulling. If these + # change in the YAML Snapcraft will consider the build step dirty. + return ["node-engine", "node-package-manager"] + + @property + def _nodejs_tar(self): + if self._nodejs_tar_handle is None: + self._nodejs_tar_handle = sources.Tar( + self._nodejs_release_uri, self._npm_dir + ) + return self._nodejs_tar_handle + + @property + def _yarn_tar(self): + if self._yarn_tar_handle is None: + self._yarn_tar_handle = sources.Tar(_YARN_URL, self._npm_dir) + return self._yarn_tar_handle + + def __init__(self, name, options, project): + super().__init__(name, options, project) + self._source_package_json = os.path.join( + os.path.abspath(self.options.source), "package.json" + ) + self._npm_dir = os.path.join(self.partdir, "npm") + self._manifest = collections.OrderedDict() + self._nodejs_release_uri = get_nodejs_release( + self.options.node_engine, self.project.deb_arch + ) + self._nodejs_tar_handle = None + self._yarn_tar_handle = None + + def pull(self): + super().pull() + os.makedirs(self._npm_dir, exist_ok=True) + self._nodejs_tar.download() + if self.options.node_package_manager == "yarn": + self._yarn_tar.download() + # do the install in the pull phase to download all dependencies. + if self.options.node_package_manager == "npm": + self._npm_install(rootdir=self.sourcedir) + else: + self._yarn_install(rootdir=self.sourcedir) + + def clean_pull(self): + super().clean_pull() + + # Remove the npm directory (if any) + if os.path.exists(self._npm_dir): + shutil.rmtree(self._npm_dir) + + def build(self): + super().build() + if self.options.node_package_manager == "npm": + installed_node_packages = self._npm_install(rootdir=self.builddir) + # Copy the content of the symlink to the build directory + # LP: #1702661 + modules_dir = os.path.join(self.installdir, "lib", "node_modules") + _copy_symlinked_content(modules_dir) + else: + installed_node_packages = self._yarn_install(rootdir=self.builddir) + lock_file_path = os.path.join(self.sourcedir, "yarn.lock") + if os.path.isfile(lock_file_path): + with open(lock_file_path) as lock_file: + self._manifest["yarn-lock-contents"] = lock_file.read() + + self._manifest["node-packages"] = [ + "{}={}".format(name, installed_node_packages[name]) + for name in installed_node_packages + ] + + def _npm_install(self, rootdir): + self._nodejs_tar.provision( + self.installdir, clean_target=False, keep_tarball=True + ) + npm_cmd = ["npm"] + self.options.npm_flags + npm_install = npm_cmd + ["--cache-min=Infinity", "install"] + for pkg in self.options.node_packages: + self.run(npm_install + ["--global"] + [pkg], cwd=rootdir) + if os.path.exists(os.path.join(rootdir, "package.json")): + self.run(npm_install, cwd=rootdir) + self.run(npm_install + ["--global"], cwd=rootdir) + for target in self.options.npm_run: + self.run(npm_cmd + ["run", target], cwd=rootdir) + return self._get_installed_node_packages("npm", self.installdir) + + def _yarn_install(self, rootdir): + self._nodejs_tar.provision( + self.installdir, clean_target=False, keep_tarball=True + ) + self._yarn_tar.provision(self._npm_dir, clean_target=False, keep_tarball=True) + yarn_cmd = [os.path.join(self._npm_dir, "bin", "yarn")] + yarn_cmd.extend(self.options.npm_flags) + if "http_proxy" in os.environ: + yarn_cmd.extend(["--proxy", os.environ["http_proxy"]]) + if "https_proxy" in os.environ: + yarn_cmd.extend(["--https-proxy", os.environ["https_proxy"]]) + flags = [] + if rootdir == self.builddir: + yarn_add = yarn_cmd + ["global", "add"] + flags.extend( + [ + "--offline", + "--prod", + "--global-folder", + self.installdir, + "--prefix", + self.installdir, + ] + ) + else: + yarn_add = yarn_cmd + ["add"] + for pkg in self.options.node_packages: + self.run(yarn_add + [pkg] + flags, cwd=rootdir) + + # local packages need to be added as if they were remote, we + # remove the local package.json so `yarn add` doesn't pollute it. + if os.path.exists(self._source_package_json): + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(rootdir, "package.json")) + shutil.copy( + self._source_package_json, os.path.join(rootdir, "package.json") + ) + self.run(yarn_add + ["file:{}".format(rootdir)] + flags, cwd=rootdir) + + # npm run would require to bring back package.json + if self.options.npm_run and os.path.exists(self._source_package_json): + # The current package.json is the yarn prefilled one. + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(rootdir, "package.json")) + os.link(self._source_package_json, os.path.join(rootdir, "package.json")) + for target in self.options.npm_run: + self.run( + yarn_cmd + ["run", target], + cwd=rootdir, + env=self._build_environment(rootdir), + ) + return self._get_installed_node_packages("npm", self.installdir) + + def _get_installed_node_packages(self, package_manager, cwd): + try: + output = self.run_output( + [package_manager, "ls", "--global", "--json"], cwd=cwd + ) + except subprocess.CalledProcessError as error: + # XXX When dependencies have missing dependencies, an error like + # this is printed to stderr: + # npm ERR! peer dep missing: glob@*, required by glob-promise@3.1.0 + # retcode is not 0, which raises an exception. + output = error.output.decode(sys.getfilesystemencoding()).strip() + packages = collections.OrderedDict() + dependencies = json.loads(output, object_pairs_hook=collections.OrderedDict)[ + "dependencies" + ] + while dependencies: + key, value = dependencies.popitem(last=False) + # XXX Just as above, dependencies without version are the ones + # missing. + if "version" in value: + packages[key] = value["version"] + if "dependencies" in value: + dependencies.update(value["dependencies"]) + return packages + + def get_manifest(self): + return self._manifest + + def _build_environment(self, rootdir): + env = os.environ.copy() + if rootdir.endswith("src"): + hidden_path = os.path.join(rootdir, "node_modules", ".bin") + if env.get("PATH"): + new_path = "{}:{}".format(hidden_path, env.get("PATH")) + else: + new_path = hidden_path + env["PATH"] = new_path + return env + + +def _get_nodejs_base(node_engine, machine): + if machine not in _NODEJS_ARCHES: + raise errors.SnapcraftEnvironmentError( + "architecture not supported ({})".format(machine) + ) + return _NODEJS_BASE.format(version=node_engine, arch=_NODEJS_ARCHES[machine]) + + +def get_nodejs_release(node_engine, arch): + return _NODEJS_TMPL.format( + version=node_engine, base=_get_nodejs_base(node_engine, arch) + ) + + +def _copy_symlinked_content(modules_dir): + """Copy symlinked content. + + When running newer versions of npm, symlinks to the local tree are + created from the part's installdir to the root of the builddir of the + part (this only affects some build configurations in some projects) + which is valid when running from the context of the part but invalid + as soon as the artifacts migrate across the steps, + i.e.; stage and prime. + + If modules_dir does not exist we simply return. + """ + if not os.path.exists(modules_dir): + return + modules = [os.path.join(modules_dir, d) for d in os.listdir(modules_dir)] + symlinks = [l for l in modules if os.path.islink(l)] + for link_path in symlinks: + link_target = os.path.realpath(link_path) + os.unlink(link_path) + link_or_copy_tree(link_target, link_path) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1b1238131..c7b2a8a60 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -61,7 +61,7 @@ parts: ls -l $SNAPCRAFT_PART_INSTALL/bin/hugo node: - plugin: nodejs + plugin: x-nodejs node-packages: [postcss-cli] filesets: node: