Commit 47668f27 authored by Yori Fournier's avatar Yori Fournier
Browse files

finished the tracibility part of run step

parent 1390f490
......@@ -76,6 +76,16 @@ source ${PIPE_ROOT}/src/build.sh
# load minikube docker
load_minikube_docker
# get the long commit
checkout_commit ${PIPE_ROOT} ${STEP_NAME} ${COMMIT}
if [[ $? != 0 ]]; then return $?; fi;
cd ${PIPE_ROOT}/${PIPELINE}/${STEP_NAME}
COMMIT=`git rev-parse HEAD` # the long version of the commit
cd ${PIPE_ROOT}
# check if the image already exists
image_exists $STEP_NAME $COMMIT
WHERE=$?
......
......@@ -23,6 +23,12 @@ touch ${TMP_DIR}/parameters_checksum.sha1
# BUILD HASH
BUILD_HASH=$(docker inspect ${STEP_NAME}:${COMMIT} --format="{{.Id}}")
if [ -z ${BUILD_HASH} ];
then
echo "ERROR: you need to build the step before running it: build_step.sh <STEP_NAME> <COMMIT>"
exit 1
fi
echo "build_hash = \"${BUILD_HASH}\"" >> ${TMP_DIR}/trace.toml
# PARAM HASH (only necessary if Parameters not in GIT repo --> COMMIT)
......@@ -36,8 +42,8 @@ done
cd ${PIPE_ROOT}
PARAM_HASH=$(sha1sum ${TMP_DIR}/parameters_checksum.sha1)
PARAM_HASH=${PARAM_HASH%% *}
echo "parameters_hash = \"sha1:${PARAM_HASH}\"" >> ${TMP_DIR}/trace.toml
PARAM_HASH="sha1:${PARAM_HASH%% *}"
echo "parameters_hash = \"${PARAM_HASH}\"" >> ${TMP_DIR}/trace.toml
# ARCHIVE HASH (only necessary if ArchDescr not in GIT repo --> COMMIT)
STEPS=`./stoml ${PIPE_ROOT}/config.toml steps`
......@@ -51,98 +57,146 @@ fi
ARCHIVE_DEPS=(`./stoml ${PIPE_ROOT}/config.toml steps.${STEP_NAME}.archives`)
for archive in "${ARCHIVE_DEPS[@]}";
do
cd ${PIPE_ROOT}/${ARCHIVES}
filenames=$(find "${archive}" -type f)
if [[ "${#ARCHIVE_DEPS[@]}" != 0 ]];
then
for filename in ${filenames};
for archive in "${ARCHIVE_DEPS[@]}";
do
sha1sum ${filename} >> ${TMP_DIR}/archives_checksum.sha1
done
cd ${PIPE_ROOT}/${ARCHIVES}
filenames=$(find "${archive}" -type f)
cd ${PIPE_ROOT}
for filename in ${filenames};
do
sha1sum ${filename} >> ${TMP_DIR}/archives_checksum.sha1
done
done;
cd ${PIPE_ROOT}
ARCHIVES_HASH=$(sha1sum ${TMP_DIR}/archives_checksum.sha1)
ARCHIVES_HASH=${ARCHIVES_HASH%% *}
echo "archives_hash = \"sha1:${ARCHIVES_HASH}\"" >> ${TMP_DIR}/trace.toml
done;
ARCHIVES_HASH=$(sha1sum ${TMP_DIR}/archives_checksum.sha1)
ARCHIVES_HASH="sha1:${ARCHIVES_HASH%% *}"
else
ARCHIVES_HASH="00000000"
fi
echo "archives_hash = \"${ARCHIVES_HASH}\"" >> ${TMP_DIR}/trace.toml
# CREATE STEP HASH
STEP_HASH_ID="${BUILD_HASH}-${PARAM_HASH}-${ARCHIVES_HASH}"
STEP_HASH=$(echo "${STEP_HASH}" | sha1sum)
STEP_HASH=${STEP_HASH%% *}
echo ${STEP_HASH_ID}
echo ${STEP_HASH}
STEP_HASH=$(echo "${STEP_HASH_ID}" | sha1sum)
STEP_HASH="sha1:${STEP_HASH%% *}"
echo "step-id = \"${STEP_HASH_ID}\"" >> ${TMP_DIR}/trace.toml
echo "step-hash = \"sha1:${STEP_HASH}\"" >> ${TMP_DIR}/trace.toml
echo "step_id = \"${STEP_HASH_ID}\"" >> ${TMP_DIR}/trace.toml
echo "step_hash = \"${STEP_HASH}\"" >> ${TMP_DIR}/trace.toml
# CREATE DEPS HASH
ARTIFACTS_DEPS=("${ARTIFACTS_DIR_HASHES[@]}")
echo "art deps: ${ARTIFACTS_DEPS[@]}"
echo "nbr art deps: ${#ARTIFACTS_DEPS[@]}"
#echo "art deps: ${ARTIFACTS_DEPS[@]}"
#echo "nbr art deps: ${#ARTIFACTS_DEPS[@]}"
#
touch ${TMP_DIR}/trace_dep.toml
if [[ ! ${#ARTIFACTS_DEPS[@]} == 0 ]];
then
DEP_HASHES=()
# get the dep hash
for artifact in "${ARTIFACTS_DEPS[@]}";
do
# extract short hash from directory name
artifact_hash=${artifact: -8}
dep_hash=${artifact: -17: -9}
parent_dep_hash=${artifact: -26: -18}
dep_name=${artifact:0: -27}
echo >> ${TMP_DIR}/trace.toml
echo "[[artifacts]]" >> ${TMP_DIR}/trace.toml
echo "step_name = \"${dep_name}\"" >> ${TMP_DIR}/trace.toml
echo "step_hash = \"${dep_hash}\"" >> ${TMP_DIR}/trace.toml
echo "dep_hash = \"${parent_dep_hash}\"" >> ${TMP_DIR}/trace.toml
echo "artifacts_hash = \"${artifact_hash}\"" >> ${TMP_DIR}/trace.toml
# trace them
echo >> ${TMP_DIR}/trace_dep.toml
echo "[[artifacts_dep]]" >> ${TMP_DIR}/trace_dep.toml
echo "step_name = \"${dep_name}\"" >> ${TMP_DIR}/trace_dep.toml
echo "step_hash = \"${dep_hash}\"" >> ${TMP_DIR}/trace_dep.toml
echo "dep_hash = \"${parent_dep_hash}\"" >> ${TMP_DIR}/trace_dep.toml
echo "artifact_hash = \"${artifact_hash}\"" >> ${TMP_DIR}/trace_dep.toml
# read full_dep_hash from folder: artifact_hash.txt
full_dep_hash=$(date | sha1sum) # TO CORRECT THIS IS JUST FOR TEST [TODO]
full_dep_hash=${full_dep_hash%% *} # TO CORRECT THIS IS JUST FOR TEST [TODO]
echo "full_dep_hash = \"sha1:${full_dep_hash}\"" >> ${TMP_DIR}/trace.toml
DEP_TRACE_FILENAME=${PIPE_ROOT}/${ARTIFACTS}/${artifact}/trace.toml # should be trace.toml (but I need to reorder the writing)
full_artifact_hash=`./stoml ${DEP_TRACE_FILENAME} ${dep_name}.artifact_hash`
#full_dep_hash=$(date | sha1sum) # TO CORRECT THIS IS JUST FOR TEST [TODO]
#full_dep_hash=${full_dep_hash%% *} # TO CORRECT THIS IS JUST FOR TEST [TODO]
echo "full_artifact_hash = \"${full_artifact_hash}\"" >> ${TMP_DIR}/trace_dep.toml
DEP_HASHES=("${DEP_HASHES[@]} ${full_dep_hash}")
done;
echo "${full_artifact_hash}"
DEP_HASHES=("${DEP_HASHES[@]}" "${full_artifact_hash}")
echo "hashes: ${DEP_HASHES[@]}"
done;
echo "dep_hashes: ${DEP_HASHES[@]}"
# sort the hashes
IFS=$'\n'; ordered=($(sort <<< "${DEP_HASHES[*]}")); unset IFS;
DEPS_HASH=$(echo "${ordered}" | sha1sum)
DEPS_HASH=${DEPS_HASH%% *}
printf -v DEP_ID "%s-" "${ordered[@]}" # print the list with - separator
DEP_ID="${DEP_ID%?}" # remove last character (extra -)
DEPS_HASH=$(echo "${DEP_ID}" | sha1sum) # hash the dep_id
DEPS_HASH="sha1:${DEPS_HASH%% *}"
else
DEPS_ID="00000000"
DEPS_HASH="00000000"
fi;
echo "dep_id = \"${DEP_ID}\"" >> ${TMP_DIR}/trace.toml
echo "deps_hash = \"${DEPS_HASH}\"" >> ${TMP_DIR}/trace.toml
# SET TEMP ARTIFACT HASH
ARTIFACT_HASH="00000000"
echo "step_name: ${STEP_NAME}"
echo " dep_hash: ${DEPS_HASH}"
echo "step_hash: ${STEP_HASH}"
echo " art_hash: ${ARTIFACT_HASH}"
echo " dir name: ${STEP_NAME}-${DEPS_HASH::8}-${STEP_HASH::8}-${ARTIFACT_HASH::8}"
# echo " dir name: ${STEP_NAME}-${DEPS_HASH:5:8}-${STEP_HASH:5:8}-${ARTIFACT_HASH:5:8}"
# look in the artifacts directory if the step already exists
# run it if it does not exists
ARTIFACT_HASH=$(date | sha1sum) # TO CORRECT THIS IS JUST FOR TEST [TODO]
ARTIFACT_HASH="sha1:${ARTIFACT_HASH%% *}" # TO CORRECT THIS IS JUST FOR TEST [TODO]
# add artifact hash to the trace
echo "artifact_hash = \"${ARTIFACT_HASH}\"" >> ${TMP_DIR}/trace.toml
# add the dependancies to the trace
cat ${TMP_DIR}/trace_dep.toml >> ${TMP_DIR}/trace.toml
# rename the folder with the artifact short-hash
deps_hash=${DEPS_HASH#*:} # remove algo (sha1, sha256, md5 ...)
short_deps_hash=${deps_hash::8} # short form
step_hash=${STEP_HASH#*:} # remove algo (sha1, sha256, md5 ...)
short_step_hash=${step_hash::8} # short form
art_hash=${ARTIFACT_HASH#*:} # remove algo (sha1, sha256, md5 ...)
short_art_hash=${art_hash::8} # short form
STEP_DIR_NAME=${STEP_NAME}-${short_deps_hash}-${short_step_hash}-${short_art_hash}
mv ${TMP_DIR} ${PIPE_ROOT}/${ARTIFACTS}/${STEP_DIR_NAME}
echo "${STEP_DIR_NAME}"
exit 0
#PARAM_HASH=
......
......@@ -14,13 +14,21 @@ archives = ["ml_solar_plates_mock"]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
commit = "a4ce45b2"
commit = "0f33b92"
archives = []
dep = ['extract-features']
[steps.select-meaningful-plates-and-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
commit = "5fc5a7bc"
commit = "81ca31c"
archives = ["ml_solar_plates_mock"]
dep = ['extract-features', 'label-features']
[steps.extract-features-properties]
......
......@@ -30,6 +30,10 @@ function build_step() {
checkout_commit ${PIPE_ROOT} ${STEP_NAME} ${COMMIT}
if [[ $? != 0 ]]; then return $?; fi;
cd ${PIPE_ROOT}/${PIPELINE}/${STEP_NAME}
COMMIT=`git rev-parse HEAD` # the long version of the commit
cd ${PIPE_ROOT}
# check if dockerfile exists
if [ ! -e ${PATH_TO_DOCKERFILE}/Dockerfile ]
then
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment