commit b827624e166519e23e22cd7b115c3c58fe3de768 Author: Scott Little Date: Fri Sep 4 11:57:40 2020 -0400 rename cgcs-centos-repo to centos-repo Story: 2006387 Task: 36912 Depends-On: https://review.opendev.org/749974 Depends-On: https://review.opendev.org/750041 Change-Id: I745a822e88aca68c207a92f14fb5370d6e3d0050 Signed-off-by: Saul Wold Signed-off-by: Scott Little diff --git a/DependenciesReviewer.py b/DependenciesReviewer.py index 728aaf9..04e7119 100644 --- a/DependenciesReviewer.py +++ b/DependenciesReviewer.py @@ -29,7 +29,7 @@ DISTRO = SELECT["centos"] # Global variables USER = getpass.getuser() WORK = os.path.abspath("..") -REPOS = os.path.join(WORK, "cgcs-root/stx/") +REPOS = os.path.join(WORK, "root/stx/") MTOOLS = os.path.join(WORK, "stx-tools/centos-mirror-tools") class PkgInfo: diff --git a/Dockerfile b/Dockerfile index 44c4e3a..72b6dc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,7 +127,7 @@ RUN groupadd -g 751 cgts && \ COPY toCOPY/finishSetup.sh /usr/local/bin COPY toCOPY/populate_downloads.sh /usr/local/bin COPY toCOPY/generate-local-repo.sh /usr/local/bin -COPY toCOPY/generate-cgcs-centos-repo.sh /usr/local/bin +COPY toCOPY/generate-centos-repo.sh /usr/local/bin COPY toCOPY/lst_utils.sh /usr/local/bin COPY toCOPY/.inputrc /home/$MYUNAME/ COPY toCOPY/builder-constraints.txt /home/$MYUNAME/ diff --git a/README.rst b/README.rst index c1bc190..380dedb 100644 --- a/README.rst +++ b/README.rst @@ -164,7 +164,7 @@ Tweaks in the StarlingX build system. NOTE: You do not need to do the following step if you've synced the latest codebase. Go into the StarlingX build system (i.e. *another* container that hosts the -cgcs build system) and perform the following steps: +build system) and perform the following steps: Build packages/ISO creation --------------------------- @@ -293,16 +293,16 @@ Initialize the source tree. repo init -u https://opendev.org/starlingx/manifest.git -m default.xml repo sync -To generate cgcs-centos-repo +To generate centos-repo ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The cgcs-centos-repo is a set of symbolic links to the packages in the +The centos-repo is a set of symbolic links to the packages in the mirror and the mock configuration file. It is needed to create these links if this is the first build or the mirror has been updated. .. code-block:: bash - generate-cgcs-centos-repo.sh /import/mirrors/CentOS + generate-centos-repo.sh /import/mirrors/CentOS Where the argument to the script is the path of the mirror. diff --git a/buildrc b/buildrc index d170470..813c1e5 100644 --- a/buildrc +++ b/buildrc @@ -74,7 +74,7 @@ export MY_BUILD_DIR=${LOCALDISK}/loadbuild/$MYUNAME/$PROJECT export MY_WORKSPACE=$MY_BUILD_DIR export MY_LOCAL_DISK=${LOCALDISK}/designer/$MYUNAME export MY_REPO_ROOT_DIR=$MY_LOCAL_DISK/$PROJECT -export MY_REPO=$MY_REPO_ROOT_DIR/cgcs-root +export MY_REPO=$MY_REPO_ROOT_DIR/root export MY_SRC_RPM_BUILD_DIR=$MY_BUILD_DIR/rpmbuild export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE export MY_BUILD_CFG_STD=$MY_WORKSPACE/std/${MY_BUILD_ENVIRONMENT_FILE_STD} diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index d76714d..3559e1f 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -109,7 +109,7 @@ fi # The 2 categories we can divide the list of packages in the output directory: # - General hosted under "downloads" output directory. # - Puppet hosted under "downloads/puppet" output directory. -# to be populated under $MY_REPO/addons/wr-cgcs/layers/cgcs/downloads/puppet +# to be populated under $MY_REPO/downloads/puppet export DL_MIRROR_LOG_DIR="${DL_MIRROR_LOG_DIR:-./logs}" export DL_MIRROR_OUTPUT_DIR="${DL_MIRROR_OUTPUT_DIR:-./output/stx/CentOS}" diff --git a/centos-mirror-tools/starlingx_add_pkgs.sh b/centos-mirror-tools/starlingx_add_pkgs.sh index 4245e3e..fb79231 100755 --- a/centos-mirror-tools/starlingx_add_pkgs.sh +++ b/centos-mirror-tools/starlingx_add_pkgs.sh @@ -90,11 +90,11 @@ source $STARLINGX_ADD_PKGS_DIR/../toCOPY/lst_utils.sh STXTOOLS=${MY_REPO_ROOT_DIR}/stx-tools -CGCSREPO_PATH=$MY_REPO/cgcs-centos-repo/Binary -TISREPO_PATH=$MY_WORKSPACE/std/rpmbuild/RPMS -TISREPO_PATH_ARGS= -if [ -e $TISREPO_PATH/repodata/repomd.xml ]; then - TISREPO_PATH_ARGS="--repofrompath tis,$TISREPO_PATH" +LOCALREPO_PATH=$MY_REPO/centos-repo/Binary +STDREPO_PATH=$MY_WORKSPACE/std/rpmbuild/RPMS +STDREPO_PATH_ARGS= +if [ -e $STDREPO_PATH/repodata/repomd.xml ]; then + STDREPO_PATH_ARGS="--repofrompath tis,$STDREPO_PATH" fi RESULTS_LOG=downloaded.log @@ -117,14 +117,14 @@ function show_usage { Usage: $(basename $0) [ -C ] [ -l ] [ -d ] ... [ ] ... -This utility uses the cgcs-centos-repo repo, and optionally the rpmbuild/RPMS +This utility uses the centos-repo repo, and optionally the rpmbuild/RPMS repo from \$MY_WORKSPACE/std, as a baseline, downloading packages required to support the list provided at command-line. The -d option allows the user to specify a package to download, or the user can specify a downloaded RPM file that has dependencies that must be downloaded. The downloaded RPMs will be written to the appropriate location under the -\$MY_REPO/cgcs-centos-repo directory. The user should be able to differentiate +\$MY_REPO/centos-repo directory. The user should be able to differentiate the downloaded files versus symlinks pointing to a downloaded or shared mirror. In addition, this utility will record a list of downloaded RPMs in the $RESULTS_LOG @@ -200,7 +200,7 @@ REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED" REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED" REPOQUERY_LOWER_LAYER_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_LOWER_LAYER_MERGED" REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED" -REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS" +REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath local,$LOCALREPO_PATH $STDREPO_PATH_ARGS" function rpmfile_requires { @@ -268,7 +268,7 @@ function simplified_pkg { echo $pkg | sed 's/-[0-9]*:/-/' } -function pkg_in_cgcsrepo { +function pkg_in_localrepo { # # Check whether the specified package is already in the downloaded (or built) repo # @@ -331,7 +331,7 @@ function download_pkg { fi echo "Downloading $url" - rpm_path=$CGCSREPO_PATH/$arch/$(basename $relativepath) + rpm_path=$LOCALREPO_PATH/$arch/$(basename $relativepath) wget -q -O $rpm_path $url if [ $? -ne 0 ]; then @@ -341,7 +341,7 @@ function download_pkg { fi # Update repo - pushd $CGCSREPO_PATH >/dev/null + pushd $LOCALREPO_PATH >/dev/null createrepo -q -g comps.xml . if [ $? -ne 0 ]; then echo "createrepo failed... Aborting" >&2 @@ -369,7 +369,7 @@ function download_rpm_dependencies { continue fi - pkg_in_cgcsrepo $pkg && continue + pkg_in_localrepo $pkg && continue download_pkg $pkg done || exit $? } @@ -383,7 +383,7 @@ if [ -n "$RPMLIST" ]; then continue fi - pkg_in_cgcsrepo $pkg && continue + pkg_in_localrepo $pkg && continue download_pkg $pkg done || exit $? @@ -392,7 +392,7 @@ fi if [ -n "$DOWNLOAD_LIST" ]; then for df in $DOWNLOAD_LIST; do - pkg_in_cgcsrepo $df && continue + pkg_in_localrepo $df && continue download_pkg $df done fi diff --git a/release/build-context.py b/release/build-context.py index a5c9b38..46d6497 100755 --- a/release/build-context.py +++ b/release/build-context.py @@ -58,7 +58,7 @@ def load_manifest(name): def load_context(name): # Extract the workspace path and git SHA for each repo - # (cd ./cgcs-root/stx/stx-config && git checkout -f 22a60625f169202a68b524ac0126afb1d10921cd)\n + # (cd ./root/stx/stx-config && git checkout -f 22a60625f169202a68b524ac0126afb1d10921cd)\n ctx = {} if "://" in name: # Open a URL diff --git a/toCOPY/finishSetup.sh b/toCOPY/finishSetup.sh index 82826c5..b8de405 100755 --- a/toCOPY/finishSetup.sh +++ b/toCOPY/finishSetup.sh @@ -27,7 +27,7 @@ mkdir -p /localdisk/loadbuild/mock-cache sudo chmod 775 /localdisk/loadbuild/mock-cache sudo chown root:mock /localdisk/loadbuild/mock-cache ### may need to add these later. once it works will try on clean localdisk setup -# [builder@bavery-WS-DESK cgcs-root]$ history | grep mkdir +# [builder@bavery-WS-DESK root]$ history | grep mkdir # 55 mkdir -p $MY_WORKSPACE/results # 66 mkdir -p $MY_WORKSPACE/std/results/$MY_BUILD_ENVIRONMENT-std # 78 mkdir -p $MY_WORKSPACE/rt/rpmbuild/RPMS diff --git a/toCOPY/generate-centos-repo.sh b/toCOPY/generate-centos-repo.sh new file mode 100755 index 0000000..4c2bcc9 --- /dev/null +++ b/toCOPY/generate-centos-repo.sh @@ -0,0 +1,462 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2019 Intel Corporation +# + +GENERATE_CENTOS_REPO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $GENERATE_CENTOS_REPO_DIR/lst_utils.sh + +mirror_dir="" +layer_dirs="" + +CREATEREPO=$(which createrepo_c) +if [ $? -ne 0 ]; then + CREATEREPO="createrepo" +fi + +usage () { + echo + echo "Create a virtual rpm repo containing only rpms listed in various lst files." + echo "The virtual repo contains only symlinks to to previously downloaded or built rpms." + echo + echo "Usage" + echo + echo "$0 [Options] [ --mirror-dir= | ]" + echo + echo "Commin Options:" + echo " --distro=: Create repo for the designated distro." + echo " Default 'centos'" + echo " --layer=: Create a smaller repo, sufficient to build" + echo " only the given layer." + echo " Default: use the LAYER environmnet valiable, or 'all'." + echo " --mirror-dir=: Set the mirror directory. This is where" + echo " the previously downloaded rpms are located." + echo + echo "Override options: For use when working on a multi-layer change" + echo " --config-dir=: Use an alternate config directory rather than the" + echo " system defined one" + echo " --layer-inc-url=,,:" + echo " Override the url for the image include file of a lower" + echo " layer's build type. Normally the url(s) is read from" + echo " ///required_layer_iso_inc.cfg" + echo " This option can be used more than once." + echo " --layer-pkg-url=,,:" + echo " Override the url for the package list of a lower" + echo " layer's build type. Normally the url(s) is read from" + echo " ///required_layer_pkgs.cfg." + echo " This option can be used more than once." + echo " --layer-wheels-inc-url=,,:" + echo " Override the url for the image include file of a lower" + echo " layer's build type. Normally the url(s) is read from" + echo " ///required_layer_wheel_inc.cfg" + echo " This option can be used more than once." + echo " --layer-dir=: Look in provided dir for packages to link to." + echo " This option can be used more than once." + echo +} + +cleanup () { + if [ -e "${mirror_content}" ]; then + \rm -f ${mirror_content} + fi + if [ -e "${TMP_LST_DIR}" ]; then + \rm -rf ${TMP_LST_DIR} + fi +} + +trap "cleanup ; exit 1" INT + +if [ -z "$MY_REPO" ]; then + echo "\$MY_REPO is not set. Ensure you are running this script" + echo "from the container and \$MY_REPO points to the root of" + echo "your folder tree." + exit -1 +fi + +TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,layer-wheels-inc-url:,mirror-dir: -n 'generate-centos-repo' -- "$@") +if [ $? -ne 0 ]; then + echo "getopt error" + usage + exit 1 +fi +eval set -- "$TEMP" + + +while true ; do + case "$1" in + --mirror-dir) mirror_dir=$2 ; shift 2 ;; + --layer-dir) layer_dirs+=" ${2/,/ }" ; shift 2 ;; + --layer-inc-url) set_layer_image_inc_urls "${2}" ; shift 2 ;; + --layer-wheels-inc-url) set_layer_wheels_inc_urls "${2}" ; shift 2 ;; + --layer-pkg-url) set_layer_pkg_urls "${2}" ; shift 2 ;; + --config-dir) set_and_validate_config_dir "${2}"; shift 2 ;; + --distro) set_and_validate_distro "${2}"; shift 2 ;; + --layer) set_and_validate_layer "${2}"; shift 2 ;; + -h|--help) echo "help"; usage; exit 0 ;; + --) shift ; break ;; + *) usage; exit 1 ;; + esac +done + +if [ "$mirror_dir" == "" ]; then + if [ $# -ne 1 ]; then + usage + exit -1 + fi + + mirror_dir=$1 +fi + +echo "mirror_dir=${mirror_dir}" +echo "config_dir=${config_dir}" +echo "distro=${distro}" +echo "layer=${layer}" +echo +echo "layer_pkg_urls=${layer_pkg_urls[@]}" +echo +echo "layer_image_inc_urls=${layer_image_inc_urls[@]}" +echo +echo "layer_wheels_inc_urls=${layer_wheels_inc_urls[@]}" +echo + +dest_dir=$MY_REPO/centos-repo +timestamp="$(date +%F_%H%M)" +mock_cfg_file=$MY_REPO/build-tools/repo_files/mock.cfg.proto +comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml +mock_cfg_dest_file=$MY_REPO/centos-repo/mock.cfg.proto +comps_xml_dest_file=$MY_REPO/centos-repo/Binary/comps.xml + +TMP_LST_DIR=$(mktemp -d /tmp/tmp_lst_dir_XXXXXX) +mkdir -p $TMP_LST_DIR +lst_file_dir="$TMP_LST_DIR" +inc_file_dir="${dest_dir}/layer_image_inc" +wheels_file_dir="${dest_dir}/layer_wheels_inc" +build_info_file_dir="${dest_dir}/layer_build_info" + +rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst" +rpm_lst_files_rt="" +other_lst_file="other_downloads.lst" + +for template in $rpm_lst_files $other_lst_file; do + lst="$lst_file_dir/${template}" + merge_lst ${config_dir} ${distro} ${template} > ${lst} +done + +missing_rpms_file=missing.txt + +\rm -f ${missing_rpms_file} + +# Strip trailing / from mirror_dir if it was specified... +mirror_dir=$(readlink -f ${mirror_dir} | sed "s%/$%%") + +if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then + echo "The mirror ${mirror_dir} doesn't has the Binary and Source" + echo "folders. Please provide a valid mirror" + exit -1 +fi + +for layer_dir in ${layer_dirs}; do + if [ ! -d ${layer_dir} ]; then + echo "The layer-dir ${layer_dir} doesn't exist" + exit -1 + fi +done + +if [ ! -d "${dest_dir}" ]; then + mkdir -p "${dest_dir}" +fi + +for t in "Binary" "Source" ; do + target_dir=${dest_dir}/$t + if [ ! -d "$target_dir" ]; then + mkdir -p "$target_dir" + else + mv -f "$target_dir" "$target_dir-backup-$timestamp" + mkdir -p "$target_dir" + fi +done + +# +# Dowload image inc files from layer_image_inc_urls +# +\rm -rf ${inc_file_dir} +mkdir -p ${inc_file_dir} +for key in "${!layer_image_inc_urls[@]}"; do + lower_layer="${key%,*}" + inc_type="${key#*,}" + url="${layer_image_inc_urls[${key}]}" + name_from_url=$(url_to_file_name "${url}") + + if [ "${inc_type}" == "std" ]; then + ideal_name="${lower_layer}_${image_inc_from_layer_build_template}" + elif [ "${inc_type}" == "dev" ]; then + ideal_name="${lower_layer}_${dev_image_inc_from_layer_build_template}" + else + ideal_name="${lower_layer}_${inc_type}_${image_inc_from_layer_build_template}" + fi + + list="${ideal_name}" + + for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do + cp $f ${inc_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" + fi + done + + if [ ! -f ${inc_file_dir}/${list} ]; then + curl --silent --fail ${url} > ${inc_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "ERROR: Failed to download from url '${url}'" + exit 1 + fi + fi +done + +# +# Dowload wheels inc files from layer_wheels_inc_urls +# +\rm -rf ${wheels_file_dir} +mkdir -p ${wheels_file_dir} +for key in "${!layer_wheels_inc_urls[@]}"; do + lower_layer="${key%,*}" + stream="${key#*,}" + url="${layer_wheels_inc_urls[${key}]}" + name_from_url=$(url_to_file_name "${url}") + + ideal_name="${lower_layer}_${distro}_${stream}_${wheels_inc_from_layer_build_template}" + + list="${ideal_name}" + + for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do + cp $f ${wheels_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" + fi + done + + if [ ! -f ${wheels_file_dir}/${list} ]; then + curl --silent --fail ${url} > ${wheels_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "ERROR: Failed to download from url '${url}'" + exit 1 + fi + fi +done + +# +# Dowload build info files +# +build_info_from_layer_build_template="BUILD_INFO" +\rm -rf ${build_info_file_dir} +mkdir -p ${build_info_file_dir} +for key in "${!layer_image_inc_urls[@]}"; do + lower_layer="${key%,*}" + inc_type="${key#*,}" + + if [ "${inc_type}" != "std" ]; then + continue + fi + + if [ "$(basename ${layer_image_inc_urls[${key}]})" != "image.inc" ]; then + continue + fi + + url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' ) + name_from_url=$(url_to_file_name "${url}") + ideal_name="${lower_layer}_${build_info_from_layer_build_template}" + + list="${ideal_name}" + + for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do + cp $f ${build_info_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" + fi + done + + if [ ! -f ${build_info_file_dir}/${list} ]; then + curl --silent --fail ${url} > ${build_info_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "WARNING: Failed to download from url '${url}'" + fi + fi +done + + +# +# Dowload lst files from layer_pkg_urls +# +for key in "${!layer_pkg_urls[@]}"; do + lower_layer="${key%,*}" + build_type="${key#*,}" + url="${layer_pkg_urls[${key}]}" + name_from_url=$(url_to_file_name "${url}") + ideal_name="${lower_layer}_${build_type}_${rpms_from_layer_build_template}" + list="${ideal_name}" + + for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do + cp $f ${lst_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" + fi + done + + if [ ! -f ${lst_file_dir}/${list} ]; then + curl --silent --fail ${url} > ${lst_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "ERROR: Failed to download from url '${url}'" + exit 1 + fi + fi + + if [ "${build_type}" == "rt" ]; then + rpm_lst_files_rt+=" ${list}" + else + rpm_lst_files+=" ${list}" + fi + + url_type=${url%%:*} + if [ "${url_type}" == "file" ]; then + url_dir=$(dirname ${url#file://}) + if [ ! -d ${url_dir} ]; then + echo "ERROR: No such directory '${url_dir}' derived from url: ${url}" + exit 1 + fi + layer_dirs+=" ${url_dir}" + fi +done + +echo "rpm_lst_files=${rpm_lst_files}" +echo "rpm_lst_files_rt=${rpm_lst_files_rt}" +echo "layer_dirs=${layer_dirs}" + +mirror_content=$(mktemp -t centos-repo-XXXXXX) +find -L ${layer_dirs} ${mirror_dir} -type f -name '*.rpm' > ${mirror_content} + +sed_expression="" +for d in ${mirror_dir} ${layer_dirs}; do + sed_expression+=" -e s%^${d}/%%" +done + + +process_lst_file () { + local lst_file="${1}" + local dest_dir="${2}" + + grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do + if [ -z "${rpmname}" ]; then + continue + fi + + mirror_file=$(grep "/${rpmname}$" ${mirror_content} | head -n 1) + if [ -z "${mirror_file}" ]; then + echo "Error -- could not find requested ${rpmname} in ${mirror_dir}" + echo ${rpmname} >> ${missing_rpms_file} + continue + fi + + # Great, we found the file! Let's strip the mirror_dir prefix from it... + ff=$(echo ${mirror_file} | sed ${sed_expression}) + f_name=$(basename "$ff") + arch=$(echo ${f_name} | rev | cut -d '.' -f 2 | rev) + if [ "${arch}" == "src" ]; then + sub_dir="Source" + else + sub_dir="Binary/${arch}" + fi + + # Make sure we have a subdir (so we don't symlink the first file as + # the subdir name) + mkdir -p ${dest_dir}/${sub_dir} + + # Link it! + echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}" + ln -sf "${mirror_file}" "${dest_dir}/${sub_dir}/${f_name}" + if [ $? -ne 0 ]; then + echo "Failed ${mirror_file}: ln -sf \"${mirror_file}\" \"${dest_dir}/${sub_dir}\"" + fi + done +} + +for lst_file in ${rpm_lst_files} ; do + process_lst_file "${lst_file}" "${dest_dir}" || exit 1 +done + +for lst_file in ${rpm_lst_files_rt} ; do + process_lst_file "${lst_file}" "${dest_dir}/rt" || exit 1 +done + + +echo "Copying comps.xml file." + +if [ ! -f "$comps_xml_file" ]; then + echo "Cannot find comps.xml file!" + exit 1 +fi + +if [ -f "$comps_xml_dest_file" ]; then + \cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp" +fi +cp "$comps_xml_file" "$comps_xml_dest_file" + + +echo "Createing yum repodata." + +for subdir in Source Binary; do + repo_dir="${dest_dir}/${subdir}" + mkdir -p "${repo_dir}" + if [ -f "${repo_dir}/comps.xml" ]; then + ${CREATEREPO} -g "${repo_dir}/comps.xml" -d "${repo_dir}" + else + ${CREATEREPO} -d "${repo_dir}" + fi + + repo_dir="${dest_dir}/rt/${subdir}" + mkdir -p "${repo_dir}" + if [ -f "${repo_dir}/comps.xml" ]; then + ${CREATEREPO} -g "${repo_dir}/comps.xml" -d "${repo_dir}" + else + ${CREATEREPO} -d "${repo_dir}" + fi +done + + +echo "Copying mock.cfg.proto file." + +if [ ! -f "$mock_cfg_file" ]; then + echo "Cannot find mock.cfg.proto file!" + exit 1 +fi + +if [ -f "$mock_cfg_dest_file" ]; then + \cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp" +fi +cp "$mock_cfg_file" "$mock_cfg_dest_file" + + +# Populate the contents from other list files +cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do + if [ "${targettype}" == "folder" ]; then + echo "Creating folder ${item}" + mkdir -p $MY_REPO/centos-repo/Binary/${item} + fi + + if [ "${targettype}" == "file" ]; then + mkdir -p $MY_REPO/centos-repo/Binary/$(dirname ${item}) + echo "Creating symlink for $MY_REPO/centos-repo/Binary/${item}" + ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/centos-repo/Binary/${item} + fi +done + +echo "Done creating repo directory" +declare -i missing_rpms_file_count=$(wc -l ${missing_rpms_file} 2>/dev/null | awk '{print $1}') +if [ ${missing_rpms_file_count} -gt 0 ]; then + echo "WARNING: Some targets could not be found. Your repo may be incomplete." + echo "Missing targets:" + cat ${missing_rpms_file} + exit 1 +fi diff --git a/toCOPY/generate-cgcs-centos-repo.sh b/toCOPY/generate-cgcs-centos-repo.sh deleted file mode 100755 index f3dbf3c..0000000 --- a/toCOPY/generate-cgcs-centos-repo.sh +++ /dev/null @@ -1,463 +0,0 @@ -#!/bin/bash -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2019 Intel Corporation -# - -GENERATE_CGCS_CENTOS_REPO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" - -source $GENERATE_CGCS_CENTOS_REPO_DIR/lst_utils.sh - -mirror_dir="" -layer_dirs="" - -CREATEREPO=$(which createrepo_c) -if [ $? -ne 0 ]; then - CREATEREPO="createrepo" -fi - -usage () { - echo - echo "Create a virtual rpm repo containing only rpms listed in various lst files." - echo "The virtual repo contains only symlinks to to previously downloaded or built rpms." - echo - echo "Usage" - echo - echo "$0 [Options] [ --mirror-dir= | ]" - echo - echo "Commin Options:" - echo " --distro=: Create repo for the designated distro." - echo " Default 'centos'" - echo " --layer=: Create a smaller repo, sufficient to build" - echo " only the given layer." - echo " Default: use the LAYER environmnet valiable, or 'all'." - echo " --mirror-dir=: Set the mirror directory. This is where" - echo " the previously downloaded rpms are located." - echo - echo "Override options: For use when working on a multi-layer change" - echo " --config-dir=: Use an alternate config directory rather than the" - echo " system defined one" - echo " --layer-inc-url=,,:" - echo " Override the url for the image include file of a lower" - echo " layer's build type. Normally the url(s) is read from" - echo " ///required_layer_iso_inc.cfg" - echo " This option can be used more than once." - echo " --layer-pkg-url=,,:" - echo " Override the url for the package list of a lower" - echo " layer's build type. Normally the url(s) is read from" - echo " ///required_layer_pkgs.cfg." - echo " This option can be used more than once." - echo " --layer-wheels-inc-url=,,:" - echo " Override the url for the image include file of a lower" - echo " layer's build type. Normally the url(s) is read from" - echo " ///required_layer_wheel_inc.cfg" - echo " This option can be used more than once." - echo " --layer-dir=: Look in provided dir for packages to link to." - echo " This option can be used more than once." - echo -} - -cleanup () { - if [ -e "${mirror_content}" ]; then - \rm -f ${mirror_content} - fi - if [ -e "${TMP_LST_DIR}" ]; then - \rm -rf ${TMP_LST_DIR} - fi -} - -trap "cleanup ; exit 1" INT - -if [ -z "$MY_REPO" ]; then - echo "\$MY_REPO is not set. Ensure you are running this script" - echo "from the container and \$MY_REPO points to the root of" - echo "your folder tree." - exit -1 -fi - - -TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,layer-wheels-inc-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@") -if [ $? -ne 0 ]; then - echo "getopt error" - usage - exit 1 -fi -eval set -- "$TEMP" - - -while true ; do - case "$1" in - --mirror-dir) mirror_dir=$2 ; shift 2 ;; - --layer-dir) layer_dirs+=" ${2/,/ }" ; shift 2 ;; - --layer-inc-url) set_layer_image_inc_urls "${2}" ; shift 2 ;; - --layer-wheels-inc-url) set_layer_wheels_inc_urls "${2}" ; shift 2 ;; - --layer-pkg-url) set_layer_pkg_urls "${2}" ; shift 2 ;; - --config-dir) set_and_validate_config_dir "${2}"; shift 2 ;; - --distro) set_and_validate_distro "${2}"; shift 2 ;; - --layer) set_and_validate_layer "${2}"; shift 2 ;; - -h|--help) echo "help"; usage; exit 0 ;; - --) shift ; break ;; - *) usage; exit 1 ;; - esac -done - -if [ "$mirror_dir" == "" ]; then - if [ $# -ne 1 ]; then - usage - exit -1 - fi - - mirror_dir=$1 -fi - -echo "mirror_dir=${mirror_dir}" -echo "config_dir=${config_dir}" -echo "distro=${distro}" -echo "layer=${layer}" -echo -echo "layer_pkg_urls=${layer_pkg_urls[@]}" -echo -echo "layer_image_inc_urls=${layer_image_inc_urls[@]}" -echo -echo "layer_wheels_inc_urls=${layer_wheels_inc_urls[@]}" -echo - -dest_dir=$MY_REPO/cgcs-centos-repo -timestamp="$(date +%F_%H%M)" -mock_cfg_file=$MY_REPO/build-tools/repo_files/mock.cfg.proto -comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml -mock_cfg_dest_file=$MY_REPO/cgcs-centos-repo/mock.cfg.proto -comps_xml_dest_file=$MY_REPO/cgcs-centos-repo/Binary/comps.xml - -TMP_LST_DIR=$(mktemp -d /tmp/tmp_lst_dir_XXXXXX) -mkdir -p $TMP_LST_DIR -lst_file_dir="$TMP_LST_DIR" -inc_file_dir="${dest_dir}/layer_image_inc" -wheels_file_dir="${dest_dir}/layer_wheels_inc" -build_info_file_dir="${dest_dir}/layer_build_info" - -rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst" -rpm_lst_files_rt="" -other_lst_file="other_downloads.lst" - -for template in $rpm_lst_files $other_lst_file; do - lst="$lst_file_dir/${template}" - merge_lst ${config_dir} ${distro} ${template} > ${lst} -done - -missing_rpms_file=missing.txt - -\rm -f ${missing_rpms_file} - -# Strip trailing / from mirror_dir if it was specified... -mirror_dir=$(readlink -f ${mirror_dir} | sed "s%/$%%") - -if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then - echo "The mirror ${mirror_dir} doesn't has the Binary and Source" - echo "folders. Please provide a valid mirror" - exit -1 -fi - -for layer_dir in ${layer_dirs}; do - if [ ! -d ${layer_dir} ]; then - echo "The layer-dir ${layer_dir} doesn't exist" - exit -1 - fi -done - -if [ ! -d "${dest_dir}" ]; then - mkdir -p "${dest_dir}" -fi - -for t in "Binary" "Source" ; do - target_dir=${dest_dir}/$t - if [ ! -d "$target_dir" ]; then - mkdir -p "$target_dir" - else - mv -f "$target_dir" "$target_dir-backup-$timestamp" - mkdir -p "$target_dir" - fi -done - -# -# Dowload image inc files from layer_image_inc_urls -# -\rm -rf ${inc_file_dir} -mkdir -p ${inc_file_dir} -for key in "${!layer_image_inc_urls[@]}"; do - lower_layer="${key%,*}" - inc_type="${key#*,}" - url="${layer_image_inc_urls[${key}]}" - name_from_url=$(url_to_file_name "${url}") - - if [ "${inc_type}" == "std" ]; then - ideal_name="${lower_layer}_${image_inc_from_layer_build_template}" - elif [ "${inc_type}" == "dev" ]; then - ideal_name="${lower_layer}_${dev_image_inc_from_layer_build_template}" - else - ideal_name="${lower_layer}_${inc_type}_${image_inc_from_layer_build_template}" - fi - - list="${ideal_name}" - - for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do - cp $f ${inc_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" - fi - done - - if [ ! -f ${inc_file_dir}/${list} ]; then - curl --silent --fail ${url} > ${inc_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "ERROR: Failed to download from url '${url}'" - exit 1 - fi - fi -done - -# -# Dowload wheels inc files from layer_wheels_inc_urls -# -\rm -rf ${wheels_file_dir} -mkdir -p ${wheels_file_dir} -for key in "${!layer_wheels_inc_urls[@]}"; do - lower_layer="${key%,*}" - stream="${key#*,}" - url="${layer_wheels_inc_urls[${key}]}" - name_from_url=$(url_to_file_name "${url}") - - ideal_name="${lower_layer}_${distro}_${stream}_${wheels_inc_from_layer_build_template}" - - list="${ideal_name}" - - for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do - cp $f ${wheels_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" - fi - done - - if [ ! -f ${wheels_file_dir}/${list} ]; then - curl --silent --fail ${url} > ${wheels_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "ERROR: Failed to download from url '${url}'" - exit 1 - fi - fi -done - -# -# Dowload build info files -# -build_info_from_layer_build_template="BUILD_INFO" -\rm -rf ${build_info_file_dir} -mkdir -p ${build_info_file_dir} -for key in "${!layer_image_inc_urls[@]}"; do - lower_layer="${key%,*}" - inc_type="${key#*,}" - - if [ "${inc_type}" != "std" ]; then - continue - fi - - if [ "$(basename ${layer_image_inc_urls[${key}]})" != "image.inc" ]; then - continue - fi - - url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' ) - name_from_url=$(url_to_file_name "${url}") - ideal_name="${lower_layer}_${build_info_from_layer_build_template}" - - list="${ideal_name}" - - for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do - cp $f ${build_info_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" - fi - done - - if [ ! -f ${build_info_file_dir}/${list} ]; then - curl --silent --fail ${url} > ${build_info_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "WARNING: Failed to download from url '${url}'" - fi - fi -done - - -# -# Dowload lst files from layer_pkg_urls -# -for key in "${!layer_pkg_urls[@]}"; do - lower_layer="${key%,*}" - build_type="${key#*,}" - url="${layer_pkg_urls[${key}]}" - name_from_url=$(url_to_file_name "${url}") - ideal_name="${lower_layer}_${build_type}_${rpms_from_layer_build_template}" - list="${ideal_name}" - - for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do - cp $f ${lst_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" - fi - done - - if [ ! -f ${lst_file_dir}/${list} ]; then - curl --silent --fail ${url} > ${lst_file_dir}/${list} - if [ $? -ne 0 ]; then - echo "ERROR: Failed to download from url '${url}'" - exit 1 - fi - fi - - if [ "${build_type}" == "rt" ]; then - rpm_lst_files_rt+=" ${list}" - else - rpm_lst_files+=" ${list}" - fi - - url_type=${url%%:*} - if [ "${url_type}" == "file" ]; then - url_dir=$(dirname ${url#file://}) - if [ ! -d ${url_dir} ]; then - echo "ERROR: No such directory '${url_dir}' derived from url: ${url}" - exit 1 - fi - layer_dirs+=" ${url_dir}" - fi -done - -echo "rpm_lst_files=${rpm_lst_files}" -echo "rpm_lst_files_rt=${rpm_lst_files_rt}" -echo "layer_dirs=${layer_dirs}" - -mirror_content=$(mktemp -t centos-repo-XXXXXX) -find -L ${layer_dirs} ${mirror_dir} -type f -name '*.rpm' > ${mirror_content} - -sed_expression="" -for d in ${mirror_dir} ${layer_dirs}; do - sed_expression+=" -e s%^${d}/%%" -done - - -process_lst_file () { - local lst_file="${1}" - local dest_dir="${2}" - - grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do - if [ -z "${rpmname}" ]; then - continue - fi - - mirror_file=$(grep "/${rpmname}$" ${mirror_content} | head -n 1) - if [ -z "${mirror_file}" ]; then - echo "Error -- could not find requested ${rpmname} in ${mirror_dir}" - echo ${rpmname} >> ${missing_rpms_file} - continue - fi - - # Great, we found the file! Let's strip the mirror_dir prefix from it... - ff=$(echo ${mirror_file} | sed ${sed_expression}) - f_name=$(basename "$ff") - arch=$(echo ${f_name} | rev | cut -d '.' -f 2 | rev) - if [ "${arch}" == "src" ]; then - sub_dir="Source" - else - sub_dir="Binary/${arch}" - fi - - # Make sure we have a subdir (so we don't symlink the first file as - # the subdir name) - mkdir -p ${dest_dir}/${sub_dir} - - # Link it! - echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}" - ln -sf "${mirror_file}" "${dest_dir}/${sub_dir}/${f_name}" - if [ $? -ne 0 ]; then - echo "Failed ${mirror_file}: ln -sf \"${mirror_file}\" \"${dest_dir}/${sub_dir}\"" - fi - done -} - -for lst_file in ${rpm_lst_files} ; do - process_lst_file "${lst_file}" "${dest_dir}" || exit 1 -done - -for lst_file in ${rpm_lst_files_rt} ; do - process_lst_file "${lst_file}" "${dest_dir}/rt" || exit 1 -done - - -echo "Copying comps.xml file." - -if [ ! -f "$comps_xml_file" ]; then - echo "Cannot find comps.xml file!" - exit 1 -fi - -if [ -f "$comps_xml_dest_file" ]; then - \cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp" -fi -cp "$comps_xml_file" "$comps_xml_dest_file" - - -echo "Createing yum repodata." - -for subdir in Source Binary; do - repo_dir="${dest_dir}/${subdir}" - mkdir -p "${repo_dir}" - if [ -f "${repo_dir}/comps.xml" ]; then - ${CREATEREPO} -g "${repo_dir}/comps.xml" -d "${repo_dir}" - else - ${CREATEREPO} -d "${repo_dir}" - fi - - repo_dir="${dest_dir}/rt/${subdir}" - mkdir -p "${repo_dir}" - if [ -f "${repo_dir}/comps.xml" ]; then - ${CREATEREPO} -g "${repo_dir}/comps.xml" -d "${repo_dir}" - else - ${CREATEREPO} -d "${repo_dir}" - fi -done - - -echo "Copying mock.cfg.proto file." - -if [ ! -f "$mock_cfg_file" ]; then - echo "Cannot find mock.cfg.proto file!" - exit 1 -fi - -if [ -f "$mock_cfg_dest_file" ]; then - \cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp" -fi -cp "$mock_cfg_file" "$mock_cfg_dest_file" - - -# Populate the contents from other list files -cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do - if [ "${targettype}" == "folder" ]; then - echo "Creating folder ${item}" - mkdir -p $MY_REPO/cgcs-centos-repo/Binary/${item} - fi - - if [ "${targettype}" == "file" ]; then - mkdir -p $MY_REPO/cgcs-centos-repo/Binary/$(dirname ${item}) - echo "Creating symlink for $MY_REPO/cgcs-centos-repo/Binary/${item}" - ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/cgcs-centos-repo/Binary/${item} - fi -done - -echo "Done creating repo directory" -declare -i missing_rpms_file_count=$(wc -l ${missing_rpms_file} 2>/dev/null | awk '{print $1}') -if [ ${missing_rpms_file_count} -gt 0 ]; then - echo "WARNING: Some targets could not be found. Your repo may be incomplete." - echo "Missing targets:" - cat ${missing_rpms_file} - exit 1 -fi diff --git a/toCOPY/generate-local-repo.sh b/toCOPY/generate-local-repo.sh index 4b782ae..d6194ba 100755 --- a/toCOPY/generate-local-repo.sh +++ b/toCOPY/generate-local-repo.sh @@ -12,7 +12,7 @@ export PATH="${MY_REPO}/build-tools:${PATH}" EXPORTS="$MY_WORKSPACE/export/mirrors/" DISTRO="CentOS" -HEAD_FILE="${MY_REPO_ROOT_DIR}/.repo/projects/cgcs-root.git/HEAD" +HEAD_FILE="${MY_REPO_ROOT_DIR}/.repo/projects/root.git/HEAD" STAT="$(stat -c %Y ${HEAD_FILE})" TIMESTAMP=$(date -d @$(stat -c %Y ${HEAD_FILE}) +%Y-%m-%dT%H:%M:%S) STD_PKG_DIR="${MY_WORKSPACE}/std/rpmbuild/RPMS"