rework clonedev to preserve order of targets
This commit is contained in:
parent
6db984ecf6
commit
5f558a3046
|
|
@ -1,30 +1,30 @@
|
|||
lookups:
|
||||
# - name: "gitea_owned_repos"
|
||||
# api_url: "https://git.snaile.de/api/v1"
|
||||
# endpoint: "repos"
|
||||
# url_field: "ssh_url"
|
||||
# token_cmd: "rbw get 'Gitea Snailed' -f 'Clonedev PAT'"
|
||||
# block_unsorted: false
|
||||
# targets:
|
||||
# - "orgs/snailed"
|
||||
# - "users/luca"
|
||||
# extra_headers:
|
||||
# - "Authorization: token ${TOKEN}"
|
||||
# - name: "github_starred_repos"
|
||||
# api_url: "https://api.github.com"
|
||||
# endpoint: "starred"
|
||||
# url_field: "ssh_url"
|
||||
# token_cmd: "rbw get 'Github' -f 'Clonedev PAT'"
|
||||
# block_unsorted: true
|
||||
# targets:
|
||||
# - "users/ssnailed"
|
||||
# extra_headers:
|
||||
# - "Authorization: Bearer ${TOKEN}"
|
||||
- name: "gitea_owned_repos"
|
||||
api_url: "https://git.snaile.de/api/v1"
|
||||
endpoint: "repos"
|
||||
url_field: "ssh_url"
|
||||
token_cmd: "rbw get 'Gitea Snailed' -f 'Clonedev PAT'"
|
||||
block_unsorted: false
|
||||
targets:
|
||||
- "orgs/snailed"
|
||||
- "users/luca"
|
||||
extra_headers:
|
||||
- "Authorization: token ${TOKEN}"
|
||||
- name: "github_starred_repos"
|
||||
api_url: "https://api.github.com"
|
||||
endpoint: "starred"
|
||||
url_field: "ssh_url"
|
||||
token_cmd: "rbw get 'Github' -f 'Clonedev PAT'"
|
||||
block_unsorted: true
|
||||
targets:
|
||||
- "users/ssnailed"
|
||||
extra_headers:
|
||||
- "Authorization: Bearer ${TOKEN}"
|
||||
- name: "tralios_gitlab"
|
||||
api_url: "https://gitlab.tralios.de/api/v4"
|
||||
targets:
|
||||
- "groups/197" # ansible-galaxy
|
||||
- "groups/73" # infrastruktur
|
||||
- "groups/197" # ansible-galaxy
|
||||
- "groups/26" # kubernetes
|
||||
- "groups/14" # docker
|
||||
endpoint: "projects"
|
||||
|
|
@ -81,8 +81,8 @@ directories:
|
|||
lookups:
|
||||
- "tralios_gitlab"
|
||||
repos:
|
||||
- "infrastruktur/ansible"
|
||||
- "kubernetes/helm"
|
||||
- "infrastruktur/ansible"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/tralios/docker"
|
||||
lookups:
|
||||
|
|
|
|||
|
|
@ -3,18 +3,21 @@
|
|||
use strict;
|
||||
use warnings;
|
||||
use feature ("signatures");
|
||||
use YAML();
|
||||
use JSON();
|
||||
use JSON;
|
||||
use WWW::Curl::Easy;
|
||||
use WWW::Curl::Multi;
|
||||
use File::Path("make_path");
|
||||
|
||||
use Data::Dumper;
|
||||
$Data::Dumper::Pair = " : ";
|
||||
$Data::Dumper::Indent = 2;
|
||||
|
||||
# TODO: Async the git clones
|
||||
use YAML;
|
||||
$YAML::Preserve = 1;
|
||||
|
||||
# TODO: Show hook/clone output in a prettier fashion (like docker buildx)
|
||||
# TODO: Allow branch selection
|
||||
# TODO: Add flags to allow checking for unclean trees, only cloning, only pulling
|
||||
# TODO: Check if directories are empty before cloning any repos to allow cloning in any order
|
||||
|
||||
use constant USERAGENT => "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)";
|
||||
use constant URL_REGEX =>
|
||||
|
|
@ -59,37 +62,13 @@ sub add_callback($handle, $callback) {
|
|||
}
|
||||
|
||||
|
||||
sub exec_callbacks($handle) {
|
||||
if ($handles[$handle]{callbacks}) {
|
||||
for my $callback (@{$handles[$handle]->{callbacks}}) {
|
||||
$callback->($handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub exec_multicurl() {
|
||||
my $curlm = WWW::Curl::Multi->new;
|
||||
for my $handle (@handles) {
|
||||
if ($handle) {
|
||||
$curlm->add_handle($handle->{curl});
|
||||
$active_requests++;
|
||||
}
|
||||
}
|
||||
while ($active_requests) {
|
||||
my $active_transfers = $curlm->perform;
|
||||
if ($active_transfers != $active_requests) {
|
||||
while (my ($handle, $ret) = $curlm->info_read) {
|
||||
if ($handle) {
|
||||
$active_requests--;
|
||||
exec_callbacks($handle);
|
||||
|
||||
# TODO: proper error checking
|
||||
# $handles[$handle]{curl}->getinfo(CURLINFO_HTTP_CODE);
|
||||
delete $handles[$handle];
|
||||
}
|
||||
}
|
||||
}
|
||||
sub exec_curl($handle) {
|
||||
my $curl = $handles[$handle]{curl};
|
||||
$curl->perform;
|
||||
my $status = $curl->getinfo(CURLINFO_HTTP_CODE);
|
||||
if ($status < 200 || $status > 300) {
|
||||
my $url = $curl->getinfo(CURLOPT_URL);
|
||||
error("Curl on $url failed with code $status");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -159,7 +138,6 @@ sub process_urls($handle) {
|
|||
|| grep(qr/$conf{lookups}[ $handles[$handle]{lookup} ]{name}/, @{$directory->{lookups}}))
|
||||
) {
|
||||
$repo{path} = `printf $directory->{path}/$repodir`;
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -181,7 +159,7 @@ sub process_urls($handle) {
|
|||
}
|
||||
|
||||
|
||||
sub dump($handle) {
|
||||
sub dump_mem($handle) {
|
||||
print("------ Handle $handle ------\n");
|
||||
print Dumper($handles[$handle]->{memory});
|
||||
}
|
||||
|
|
@ -208,7 +186,7 @@ sub handle_repos($handle) {
|
|||
} elsif (!folder_is_empty("$repo->{path}/.git")) {
|
||||
info("Pulling $repo->{fullname} to $repo->{path}");
|
||||
if (`git -C $repo->{path} status -z`) {
|
||||
warn("$repo->{path} has an unclean tree.");
|
||||
warning("$repo->{path} has an unclean tree.");
|
||||
} else {
|
||||
`git -C $repo->{path} pull $conf{pull_flags}`;
|
||||
}
|
||||
|
|
@ -236,18 +214,17 @@ sub read_conf() {
|
|||
if (!$hashref->{hook_dir}) {
|
||||
$hashref->{hook_dir} = "$configdir/hooks";
|
||||
}
|
||||
|
||||
%conf = %$hashref;
|
||||
}
|
||||
|
||||
|
||||
sub curl_pipeline($handle) {
|
||||
add_callback($handle, \&json_decode);
|
||||
add_callback($handle, \&url_filter);
|
||||
add_callback($handle, \&process_urls);
|
||||
|
||||
# add_callback($handle, \&dump);
|
||||
add_callback($handle, \&handle_repos);
|
||||
sub full_pipeline($handle, $url, @headers) {
|
||||
set_curl($handle, $url, @headers);
|
||||
exec_curl($handle);
|
||||
json_decode($handle);
|
||||
url_filter($handle);
|
||||
process_urls($handle);
|
||||
handle_repos($handle);
|
||||
}
|
||||
|
||||
# ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░
|
||||
|
|
@ -272,24 +249,19 @@ for my $i (keys @{$conf{lookups}}) {
|
|||
for my $j (keys @{$lookup{targets}}) {
|
||||
$last_handle++;
|
||||
$handles[$last_handle]{lookup} = $i;
|
||||
set_curl($last_handle, "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", @{$lookup{extra_headers}});
|
||||
curl_pipeline($last_handle);
|
||||
full_pipeline($last_handle, "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", @{$lookup{extra_headers}});
|
||||
}
|
||||
} else {
|
||||
$last_handle++;
|
||||
$handles[$last_handle]{lookup} = $i;
|
||||
set_curl($last_handle, "$lookup{api_url}/$lookup{endpoint}", @{$lookup{extra_headers}});
|
||||
curl_pipeline($last_handle);
|
||||
full_pipeline($last_handle, "$lookup{api_url}/$lookup{endpoint}", @{$lookup{extra_headers}});
|
||||
}
|
||||
}
|
||||
|
||||
exec_multicurl();
|
||||
|
||||
$last_handle++;
|
||||
add_callback($last_handle, \&inject_conf_urls);
|
||||
add_callback($last_handle, \&process_urls);
|
||||
add_callback($last_handle, \&handle_repos);
|
||||
exec_callbacks($last_handle);
|
||||
inject_conf_urls($last_handle);
|
||||
process_urls($last_handle);
|
||||
handle_repos($last_handle);
|
||||
|
||||
for my $message (@messages) {
|
||||
print($message);
|
||||
|
|
|
|||
Loading…
Reference in New Issue