This commit is contained in:
Felix Lohmeier
GitHub
parent
db73bca422
commit
b5ee345a59
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/bash
|
||||
# openrefine-bash-curl.sh, Felix Lohmeier, v0.4.1, 2020-07-04
|
||||
# openrefine-bash-curl.sh, Felix Lohmeier, v0.5, 2020-07-07
|
||||
# How to control OpenRefine 3.3+ with cURL (and jq) in Bash scripts
|
||||
# https://gist.github.com/felixlohmeier/d76bd27fbc4b8ab6d683822cdf61f81d
|
||||
# tested on Linux (Fedora 33), needs to be adapted to work on macOS
|
||||
|
|
@ -131,6 +131,9 @@ exec &> >(tee -a "${workspace}/${date}.log")
|
|||
function log() {
|
||||
echo "$(date +%H:%M:%S.%3N) [ client] $1"
|
||||
}
|
||||
function error() {
|
||||
echo 1>&2 "ERROR: $1"; stop; exit 1
|
||||
}
|
||||
|
||||
# ======================= TEMPLATES FOR YOUR WORKFLOW ======================== #
|
||||
|
||||
|
|
@ -163,11 +166,12 @@ $ \ '
|
|||
DATA
|
||||
then
|
||||
store "${p}" "${workspace}/${filename}.id" \
|
||||
|| { echo 1>&2 "ERROR: import of ${input} failed!"; stop; exit 1; } \
|
||||
&& log "imported ${input} as ${p} (${ids[$p]})"; echo
|
||||
|| error "import of ${input} failed!" \
|
||||
&& log "imported ${input} as ${p} (${ids[$p]})"
|
||||
else
|
||||
echo 1>&2 "ERROR: import of ${input} failed!"; stop; exit 1
|
||||
error "import of ${input} failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- IMPORT OPTION 2 ------------------------------ #
|
||||
|
||||
|
|
@ -193,11 +197,12 @@ if curl -fsS --write-out "%{redirect_url}\n" \
|
|||
> "${workspace}/${filename}.id"
|
||||
then
|
||||
store "${p}" "${workspace}/${filename}.id" \
|
||||
|| { echo 1>&2 "ERROR: import of ${input} failed!"; stop; exit 1; } \
|
||||
&& log "imported ${input} as ${p} (${ids[$p]})"; echo
|
||||
|| error "import of ${input} failed!" \
|
||||
&& log "imported ${input} as ${p} (${ids[$p]})"
|
||||
else
|
||||
echo 1>&2 "ERROR: import of ${input} failed!"; stop; exit 1
|
||||
error "import of ${input} failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- IMPORT OPTION 3 ------------------------------ #
|
||||
|
||||
|
|
@ -231,10 +236,10 @@ for i in "${!inputs[@]}"; do
|
|||
wait "${pid[$i]}"
|
||||
if [[ $(wait "${pid[$i]}") -eq 0 ]]; then
|
||||
store "${p}" "${workspace}/${filename}.id" \
|
||||
|| { echo 1>&2 "ERROR: import of ${input} failed!"; stop; exit 1; } \
|
||||
|| error "import of ${input} failed!" \
|
||||
&& log "imported ${inputs[$i]} as ${p} (${ids[$p]})"
|
||||
else
|
||||
echo 1>&2 "ERROR: import of ${inputs[$i]} failed!"; stop; exit 1
|
||||
error "import of ${inputs[$i]} failed!"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
|
@ -268,11 +273,10 @@ if curl -fsS \
|
|||
"${endpoint}/command/core/apply-operations?csrf_token=$(csrf)" > /dev/null
|
||||
then
|
||||
log "transformed ${p} (${ids[$p]}) with ${input}"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: transform ${p} (${ids[$p]}) with ${input} failed!"
|
||||
stop; exit 1
|
||||
error "transform ${p} (${ids[$p]}) with ${input} failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ---------------------------- TRANSFORM OPTION 2 ---------------------------- #
|
||||
|
||||
|
|
@ -300,10 +304,10 @@ if curl -fsS \
|
|||
JSON
|
||||
then
|
||||
log "transformed ${p} (${ids[$p]})"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: transform ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "transform ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ---------------------------- TRANSFORM OPTION 3 ---------------------------- #
|
||||
|
||||
|
|
@ -334,10 +338,10 @@ if curl -fsS \
|
|||
JSON
|
||||
then
|
||||
log "transformed ${p} (${ids[$p]})"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: transform ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "transform ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ---------------------------- TRANSFORM OPTION 4 ---------------------------- #
|
||||
|
||||
|
|
@ -372,10 +376,10 @@ if curl -fsS \
|
|||
JSON
|
||||
then
|
||||
log "transformed ${p} (${ids[$p]})"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: transform ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "transform ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ---------------------------- TRANSFORM OPTION 5 ---------------------------- #
|
||||
|
||||
|
|
@ -401,10 +405,10 @@ if echo "${payload[@]}" | "${jq}" -s add | curl -fsS \
|
|||
"${endpoint}/command/core/apply-operations?csrf_token=$(csrf)" > /dev/null
|
||||
then
|
||||
log "transformed ${p} (${ids[$p]})"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: transform ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "transform ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- EXPORT OPTION 1 ------------------------------ #
|
||||
|
||||
|
|
@ -418,10 +422,11 @@ if curl -fsS \
|
|||
"${endpoint}/command/core/export-rows"
|
||||
then
|
||||
#log "printed export of ${p} (${ids[$p]})"
|
||||
echo
|
||||
:
|
||||
else
|
||||
echo 1>&2 "ERROR: export of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "export of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- EXPORT OPTION 2 ------------------------------ #
|
||||
|
||||
|
|
@ -437,10 +442,10 @@ if curl -fsS \
|
|||
> "${output}"
|
||||
then
|
||||
log "${p} (${ids[$p]}) saved to file ${output}"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: export of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "export of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- EXPORT OPTION 3 ------------------------------ #
|
||||
|
||||
|
|
@ -468,10 +473,10 @@ if echo "${template}" | head -c -2 | curl -fsS \
|
|||
then
|
||||
echo
|
||||
#log "printed export of ${p} (${ids[$p]})"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: export of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "export of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- EXPORT OPTION 4 ------------------------------ #
|
||||
|
||||
|
|
@ -500,10 +505,10 @@ if echo "${template}" | head -c -2 | curl -fsS \
|
|||
> "${output}"
|
||||
then
|
||||
log "${p} (${ids[$p]}) saved to ${output}"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: export of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "export of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ----------------------------- EXPORT OPTION 5 ------------------------------ #
|
||||
|
||||
|
|
@ -523,10 +528,11 @@ for p in "${ps[@]}"; do
|
|||
done
|
||||
for i in "${!ps[@]}"; do
|
||||
p="${ps[$i]}"
|
||||
wait "${pid[$i]}"
|
||||
if [[ $(wait "${pid[$i]}") -eq 0 ]]; then
|
||||
log "${p} (${ids[$p]}) saved to ${workspace}/${p}.${format}"
|
||||
else
|
||||
echo 1>&2 "ERROR: export of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "export of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
|
@ -540,10 +546,11 @@ if curl -fsS --get \
|
|||
| "${jq}" -r '.projects | keys[] as $k | "\($k): \(.[$k] | .name)"'
|
||||
then
|
||||
#log "printed list of projects"
|
||||
echo
|
||||
:
|
||||
else
|
||||
echo 1>&2 "ERROR: list projects failed!"; stop; exit 1
|
||||
error "list projects failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ------------------------------- GET METADATA ------------------------------- #
|
||||
|
||||
|
|
@ -556,10 +563,11 @@ if curl -fsS --get \
|
|||
| "${jq}" "{ id: ${ids[$p]} } + ."
|
||||
then
|
||||
#log "printed metadata of ${p} (${ids[$p]})"
|
||||
echo
|
||||
:
|
||||
else
|
||||
echo 1>&2 "ERROR: getting metadata of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "getting metadata of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ------------------------------ GET ROW COUNT ------------------------------- #
|
||||
|
||||
|
|
@ -572,10 +580,11 @@ if curl -fsS --get \
|
|||
| "${jq}" -r '.total'
|
||||
then
|
||||
#log "printed row count of ${p} (${ids[$p]})"
|
||||
echo
|
||||
:
|
||||
else
|
||||
echo 1>&2 "ERROR: getting rowcount of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "getting rowcount of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ------------------------------- GET COLUMNS -------------------------------- #
|
||||
|
||||
|
|
@ -588,10 +597,11 @@ if curl -fsS --get \
|
|||
| "${jq}" -r '.columnModel | .columns[] | .name'
|
||||
then
|
||||
#log "printed column names of ${p} (${ids[$p]})"
|
||||
echo
|
||||
:
|
||||
else
|
||||
echo 1>&2 "ERROR: getting columns of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "getting columns of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# -------------------------- GET OPERATIONS HISTORY -------------------------- #
|
||||
|
||||
|
|
@ -606,11 +616,10 @@ if curl -fsS --get \
|
|||
> "${output}"
|
||||
then
|
||||
log "ops history of ${p} (${ids[$p]}) saved to ${output}"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: getting ops history of ${p} (${ids[$p]}) failed!"
|
||||
stop; exit 1
|
||||
error "getting ops history of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ---------------------------- GET IMPORT HISTORY ---------------------------- #
|
||||
|
||||
|
|
@ -623,11 +632,11 @@ if curl -fsS --get \
|
|||
| "${jq}" ".importOptionMetadata[0]"
|
||||
then
|
||||
#log "printed import history of ${p} (${ids[$p]})"
|
||||
echo
|
||||
:
|
||||
else
|
||||
echo 1>&2 "ERROR: getting imp history of ${p} (${ids[$p]}) failed!"
|
||||
stop; exit 1
|
||||
error "getting imp history of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ---------------------------------- DELETE ---------------------------------- #
|
||||
|
||||
|
|
@ -639,10 +648,10 @@ if curl -fsS \
|
|||
"${endpoint}/command/core/delete-project?csrf_token=$(csrf)" > /dev/null
|
||||
then
|
||||
log "deleted ${p} (${ids[$p]})"
|
||||
echo
|
||||
else
|
||||
echo 1>&2 "ERROR: deletion of ${p} (${ids[$p]}) failed!"; stop; exit 1
|
||||
error "deletion of ${p} (${ids[$p]}) failed!"
|
||||
fi
|
||||
echo
|
||||
|
||||
# ------------------------------- STOP SERVER -------------------------------- #
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue