add path transformation and bugfix clonedev
This commit is contained in:
parent
d900ed8964
commit
0b80f982bb
|
|
@ -1,34 +1,38 @@
|
|||
# token_cmd: "secret-tool lookup gitea api_token"
|
||||
lookups:
|
||||
- name: "gitea_owned_repos"
|
||||
api_url: "https://git.snaile.de/api/v1"
|
||||
endpoint: "repos"
|
||||
url_field: "ssh_url"
|
||||
token_cmd: "rbw get 'Gitea Snailed' -f 'Clonedev PAT'"
|
||||
block_unsorted: false
|
||||
targets:
|
||||
- "orgs/snailed"
|
||||
- "users/luca"
|
||||
extra_headers:
|
||||
- "Authorization: token ${TOKEN}"
|
||||
- name: "github_starred_repos"
|
||||
api_url: "https://api.github.com"
|
||||
endpoint: "starred"
|
||||
url_field: "ssh_url"
|
||||
token_cmd: "rbw get 'Github' -f 'Clonedev PAT'"
|
||||
block_unsorted: true
|
||||
targets:
|
||||
- "users/ssnailed"
|
||||
extra_headers:
|
||||
- "Authorization: Bearer ${TOKEN}"
|
||||
# - name: "tralios_gitlab"
|
||||
# api_url: "https://gitlab.tralios.de/api/v4"
|
||||
# endpoint: "projects"
|
||||
# url_field: "ssh_url_to_repo"
|
||||
# token_cmd: "rbw get 'Gitlab Tralios' -f 'Clonedev PAT'"
|
||||
# - name: "gitea_owned_repos"
|
||||
# api_url: "https://git.snaile.de/api/v1"
|
||||
# endpoint: "repos"
|
||||
# url_field: "ssh_url"
|
||||
# token_cmd: "rbw get 'Gitea Snailed' -f 'Clonedev PAT'"
|
||||
# block_unsorted: false
|
||||
# targets:
|
||||
# - "orgs/snailed"
|
||||
# - "users/luca"
|
||||
# extra_headers:
|
||||
# - "Authorization: token ${TOKEN}"
|
||||
# - name: "github_starred_repos"
|
||||
# api_url: "https://api.github.com"
|
||||
# endpoint: "starred"
|
||||
# url_field: "ssh_url"
|
||||
# token_cmd: "rbw get 'Github' -f 'Clonedev PAT'"
|
||||
# block_unsorted: true
|
||||
# targets:
|
||||
# - "users/ssnailed"
|
||||
# extra_headers:
|
||||
# - "Authorization: Bearer ${TOKEN}"
|
||||
- name: "tralios_gitlab"
|
||||
api_url: "https://gitlab.tralios.de/api/v4"
|
||||
targets:
|
||||
- "groups/197" # ansible-galaxy
|
||||
- "groups/73" # infrastruktur
|
||||
- "groups/26" # kubernetes
|
||||
- "groups/14" # docker
|
||||
endpoint: "projects"
|
||||
url_field: "ssh_url_to_repo"
|
||||
token_cmd: "rbw get 'Gitlab Tralios' -f 'Clonedev PAT'"
|
||||
block_unsorted: true
|
||||
extra_headers:
|
||||
- "Authorization: Bearer ${TOKEN}"
|
||||
|
||||
extra_urls:
|
||||
# - "https://git.kernel.org/pub/scm/git/git.git"
|
||||
|
|
@ -46,17 +50,21 @@ directories:
|
|||
- "luca/st-custom"
|
||||
- "luca/tokyonight-icons"
|
||||
- "luca/tokyonight-theme"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/nvim"
|
||||
repos:
|
||||
- "snailed/taolf"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/misc"
|
||||
repos:
|
||||
- "snailed/tidal-scraper"
|
||||
- "b3nj5m1n/xdg-ninja"
|
||||
- "jnweiger/led-name-badge-ls32"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev"
|
||||
repos:
|
||||
- "void-linux/void-packages"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/server"
|
||||
repos:
|
||||
- "snailed/ansible-example"
|
||||
|
|
@ -69,6 +77,32 @@ directories:
|
|||
- "luca/dotfiles-server"
|
||||
- "spantaleev/matrix-docker-ansible-deploy"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/tralios"
|
||||
lookups:
|
||||
- "tralios_gitlab"
|
||||
repos:
|
||||
- "infrastruktur/ansible"
|
||||
- "kubernetes/helm"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/tralios/docker"
|
||||
lookups:
|
||||
- "tralios_gitlab"
|
||||
repos:
|
||||
- "docker/phpldapadmin"
|
||||
- "docker/nextcloud_mail_uploader"
|
||||
|
||||
- path: "${XDG_DOCUMENTS_DIR}/dev/tralios/ansible/collections_dev/ansible_collections/tralios"
|
||||
lookups:
|
||||
- "tralios_gitlab"
|
||||
repos:
|
||||
- "ansible-galaxy/kubernetes.*"
|
||||
|
||||
# Transforms the directory name that is cloned into (only the last part of the path)
|
||||
path_transformations:
|
||||
- match: "ansible-galaxy/kubernetes.*"
|
||||
replace: "-"
|
||||
with: "_"
|
||||
|
||||
unsorted_directory: "${XDG_DOCUMENTS_DIR}/dev/unsorted"
|
||||
|
||||
skip_repos:
|
||||
|
|
|
|||
|
|
@ -17,63 +17,73 @@ $Data::Dumper::Indent = 2;
|
|||
# TODO: Show hook/clone output in a prettier fashion (like docker buildx)
|
||||
|
||||
use constant USERAGENT => "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)";
|
||||
use constant URL_REGEX =>
|
||||
qr/^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/;
|
||||
|
||||
my @handles;
|
||||
my @messages;
|
||||
my %conf;
|
||||
my $active_repos = 0;
|
||||
my $active_repos = 0;
|
||||
my $active_requests = 0;
|
||||
|
||||
|
||||
sub info($message) {
|
||||
print(`tput setaf 4; tput bold; echo '$message'; tput sgr 0`);
|
||||
}
|
||||
|
||||
|
||||
sub error($message) {
|
||||
push( @messages, `tput setaf 1; tput bold; echo '$message'; tput sgr 0` );
|
||||
push(@messages, `tput setaf 1; tput bold; echo '$message'; tput sgr 0`);
|
||||
}
|
||||
|
||||
|
||||
sub warning($message) {
|
||||
push( @messages, `tput setaf 3; tput bold; echo '$message'; tput sgr 0` );
|
||||
push(@messages, `tput setaf 3; tput bold; echo '$message'; tput sgr 0`);
|
||||
}
|
||||
|
||||
sub set_curl( $handle, $url, @headers ) {
|
||||
|
||||
sub set_curl($handle, $url, @headers) {
|
||||
$handles[$handle]{curl} = WWW::Curl::Easy->new;
|
||||
$handles[$handle]{curl}->setopt( CURLOPT_URL, $url );
|
||||
$handles[$handle]{curl}->pushopt( CURLOPT_HTTPHEADER, [USERAGENT] );
|
||||
$handles[$handle]{curl}->setopt(CURLOPT_URL, $url);
|
||||
$handles[$handle]{curl}->pushopt(CURLOPT_HTTPHEADER, [USERAGENT]);
|
||||
for my $header (@headers) {
|
||||
$handles[$handle]{curl}->pushopt( CURLOPT_HTTPHEADER, [$header] );
|
||||
$handles[$handle]{curl}->pushopt(CURLOPT_HTTPHEADER, [$header]);
|
||||
}
|
||||
$handles[$handle]{curl}->setopt( CURLOPT_PRIVATE, $handle );
|
||||
$handles[$handle]{curl}->setopt( CURLOPT_WRITEDATA, \$handles[$handle]{memory} );
|
||||
$handles[$handle]{curl}->setopt(CURLOPT_PRIVATE, $handle);
|
||||
$handles[$handle]{curl}->setopt(CURLOPT_WRITEDATA, \$handles[$handle]{memory});
|
||||
}
|
||||
|
||||
sub add_callback( $handle, $callback ) {
|
||||
push( @{ $handles[$handle]{callbacks} }, $callback );
|
||||
|
||||
sub add_callback($handle, $callback) {
|
||||
push(@{$handles[$handle]{callbacks}}, $callback);
|
||||
}
|
||||
|
||||
|
||||
sub exec_callbacks($handle) {
|
||||
if ( $handles[$handle]{callbacks} ) {
|
||||
for my $callback ( @{ $handles[$handle]->{callbacks} } ) {
|
||||
if ($handles[$handle]{callbacks}) {
|
||||
for my $callback (@{$handles[$handle]->{callbacks}}) {
|
||||
$callback->($handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub exec_multicurl() {
|
||||
my $curlm = WWW::Curl::Multi->new;
|
||||
for my $handle (@handles) {
|
||||
if ($handle) {
|
||||
$curlm->add_handle( $handle->{curl} );
|
||||
$curlm->add_handle($handle->{curl});
|
||||
$active_requests++;
|
||||
}
|
||||
}
|
||||
while ($active_requests) {
|
||||
my $active_transfers = $curlm->perform;
|
||||
if ( $active_transfers != $active_requests ) {
|
||||
while ( my ( $handle, $ret ) = $curlm->info_read ) {
|
||||
if ($active_transfers != $active_requests) {
|
||||
while (my ($handle, $ret) = $curlm->info_read) {
|
||||
if ($handle) {
|
||||
$active_requests--;
|
||||
exec_callbacks($handle);
|
||||
|
||||
# TODO: proper error checking
|
||||
# $handles[$handle]{curl}->getinfo(CURLINFO_HTTP_CODE);
|
||||
delete $handles[$handle];
|
||||
|
|
@ -83,40 +93,44 @@ sub exec_multicurl() {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
sub json_decode($handle) {
|
||||
$handles[$handle]{memory} = JSON::decode_json( $handles[$handle]{memory} );
|
||||
$handles[$handle]{memory} = JSON::decode_json($handles[$handle]{memory});
|
||||
}
|
||||
|
||||
|
||||
sub url_filter($handle) {
|
||||
my @tmp;
|
||||
my $lookup = $conf{lookups}[ $handles[$handle]->{lookup} ];
|
||||
for my $repo ( @{ $handles[$handle]{memory} } ) {
|
||||
if ( $repo->{ $lookup->{url_field} } ) {
|
||||
push( @tmp, $repo->{ $lookup->{url_field} } );
|
||||
my $lookup = $conf{lookups}[$handles[$handle]->{lookup}];
|
||||
for my $repo (@{$handles[$handle]{memory}}) {
|
||||
if ($repo->{$lookup->{url_field}}) {
|
||||
push(@tmp, $repo->{$lookup->{url_field}});
|
||||
} else {
|
||||
error( "Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}" );
|
||||
error("Failed to extract $lookup->{url_field} while processing lookup: $lookup->{name}");
|
||||
}
|
||||
}
|
||||
$handles[$handle]{memory} = \@tmp;
|
||||
}
|
||||
|
||||
|
||||
sub inject_conf_urls($handle) {
|
||||
for my $url ( @{ $conf{extra_urls} } ) {
|
||||
push( @{ $handles[$handle]{memory} }, $url );
|
||||
for my $url (@{$conf{extra_urls}}) {
|
||||
push(@{$handles[$handle]{memory}}, $url);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub process_urls($handle) {
|
||||
my @tmp;
|
||||
for my $url ( @{ $handles[$handle]{memory} } ) {
|
||||
for my $url (@{$handles[$handle]{memory}}) {
|
||||
my %repo;
|
||||
if ( $url =~ /^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/ ) {
|
||||
if ($url =~ URL_REGEX) {
|
||||
$repo{url} = $1;
|
||||
$repo{owner} = $3;
|
||||
$repo{name} = $4;
|
||||
$repo{fullname} = "$3/$4";
|
||||
|
||||
if ( substr( $2, -1 ) eq "@" ) {
|
||||
if (substr($2, -1) eq "@") {
|
||||
$repo{protocol} = "ssh";
|
||||
} else {
|
||||
$repo{protocol} = $2;
|
||||
|
|
@ -126,97 +140,114 @@ sub process_urls($handle) {
|
|||
next;
|
||||
}
|
||||
|
||||
next if ( grep( $_ eq $repo{fullname}, @{ $conf{skip_repos} } ) );
|
||||
next if (grep($_ eq $repo{fullname}, @{$conf{skip_repos}}));
|
||||
|
||||
my $path;
|
||||
DIRS: for my $directory ( @{ $conf{directories} } ) {
|
||||
for my $regex ( @{ $directory->{repos} } ) {
|
||||
if ( $repo{fullname} =~ /$regex/ ) {
|
||||
$repo{path} = `printf $directory->{path}/$repo{name}`;
|
||||
last DIRS;
|
||||
|
||||
my $repodir = $repo{name};
|
||||
for my $pt (@{$conf{path_transformations}}) {
|
||||
if ($repo{fullname} =~ qr/$pt->{match}/) {
|
||||
$repodir =~ s/$pt->{replace}/$pt->{with}/;
|
||||
}
|
||||
}
|
||||
|
||||
for my $directory (@{$conf{directories}}) {
|
||||
for my $regex (@{$directory->{repos}}) {
|
||||
if (
|
||||
$repo{fullname} =~ qr/^$regex$/
|
||||
&& (!$directory->{lookups}
|
||||
|| grep(qr/$conf{lookups}[ $handles[$handle]{lookup} ]{name}/, @{$directory->{lookups}}))
|
||||
) {
|
||||
$repo{path} = `printf $directory->{path}/$repodir`;
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( !$repo{path} && !$conf{lookups}[ $handles[$handle]{lookup} ]{block_unsorted} ) {
|
||||
$repo{path} = `printf $conf{unsorted_directory}/$repo{name}`;
|
||||
if (!$repo{path} && !$conf{lookups}[$handles[$handle]{lookup}]{block_unsorted}) {
|
||||
$repo{path} = `printf $conf{unsorted_directory}/$repodir`;
|
||||
} elsif ($repo{path}) {
|
||||
} else {
|
||||
next;
|
||||
}
|
||||
|
||||
my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}";
|
||||
my $pull_hook = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}";
|
||||
( -x $clone_hook ) and $repo{clone_hook} = "cd $repo{path} && $clone_hook";
|
||||
( -x $pull_hook ) and $repo{pull_hook} = "cd $repo{path} && $pull_hook";
|
||||
push( @tmp, \%repo );
|
||||
(-x $clone_hook) and $repo{clone_hook} = "cd $repo{path} && $clone_hook";
|
||||
(-x $pull_hook) and $repo{pull_hook} = "cd $repo{path} && $pull_hook";
|
||||
push(@tmp, \%repo);
|
||||
}
|
||||
$handles[$handle]{memory} = \@tmp;
|
||||
}
|
||||
|
||||
|
||||
sub dump($handle) {
|
||||
print("------ Handle $handle ------\n");
|
||||
print Dumper( $handles[$handle]->{memory} );
|
||||
print Dumper($handles[$handle]->{memory});
|
||||
}
|
||||
|
||||
|
||||
sub folder_is_empty($directory) {
|
||||
opendir( my $dh, $directory ) or return 1;
|
||||
return scalar( grep( $_ ne "." && $_ ne "..", readdir($dh) ) ) == 0;
|
||||
opendir(my $dh, $directory) or return 1;
|
||||
return scalar(grep($_ ne "." && $_ ne "..", readdir($dh))) == 0;
|
||||
}
|
||||
|
||||
|
||||
sub handle_repos($handle) {
|
||||
for my $repo ( @{ $handles[$handle]->{memory} } ) {
|
||||
if ( folder_is_empty("$repo->{path}") ) {
|
||||
make_path( $repo->{path} );
|
||||
for my $repo (@{$handles[$handle]->{memory}}) {
|
||||
if (folder_is_empty("$repo->{path}")) {
|
||||
make_path($repo->{path});
|
||||
info("Cloning $repo->{fullname}");
|
||||
`git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`;
|
||||
( $? != 0 ) and error("Failed to clone $repo->{url} to $repo->{path}");
|
||||
if ( $repo->{clone_hook} ) {
|
||||
($? != 0) and error("Failed to clone $repo->{url} to $repo->{path}");
|
||||
if ($repo->{clone_hook}) {
|
||||
info("Running clone hook for $repo->{fullname}");
|
||||
`$repo->{clone_hook}`;
|
||||
}
|
||||
( $? != 0 ) and error("Failed to execute clone hook for $repo->{fullname}");
|
||||
}
|
||||
elsif ( !folder_is_empty("$repo->{path}/.git") ) {
|
||||
($? != 0) and error("Failed to execute clone hook for $repo->{fullname}");
|
||||
} elsif (!folder_is_empty("$repo->{path}/.git")) {
|
||||
info("Pulling $repo->{fullname} to $repo->{path}");
|
||||
if (`git -C $repo->{path} status -z`) {
|
||||
warn("$repo->{path} has an unclean tree.");
|
||||
} else {
|
||||
`git -C $repo->{path} pull $conf{pull_flags}`;
|
||||
}
|
||||
( $? != 0 ) and error("Failed to pull $repo->{url} to $repo->{path}");
|
||||
if ( $repo->{pull_hook} ) {
|
||||
($? != 0) and error("Failed to pull $repo->{url} to $repo->{path}");
|
||||
if ($repo->{pull_hook}) {
|
||||
info("Running pull hook for $repo->{fullname}");
|
||||
`$repo->{pull_hook}`;
|
||||
}
|
||||
( $? != 0 ) and error("Failed to execute pull hook for $repo->{fullname}");
|
||||
($? != 0) and error("Failed to execute pull hook for $repo->{fullname}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub read_conf() {
|
||||
my $configdir;
|
||||
if ( $ENV{XDG_CONFIG_HOME} ) {
|
||||
if ($ENV{XDG_CONFIG_HOME}) {
|
||||
$configdir = "$ENV{XDG_CONFIG_HOME}/clonedev";
|
||||
} else {
|
||||
$configdir = "$ENV{HOME}/.config/clonedev";
|
||||
}
|
||||
open( my $cfg, '<', $configdir . "/config.yml" ) or die;
|
||||
my $hashref = YAML::Load( do { local $/; <$cfg> } );
|
||||
open(my $cfg, '<', $configdir . "/config.yml") or die;
|
||||
my $hashref = YAML::Load(do {local $/; <$cfg>});
|
||||
close($cfg);
|
||||
if ( !$hashref->{hook_dir} ) {
|
||||
if (!$hashref->{hook_dir}) {
|
||||
$hashref->{hook_dir} = "$configdir/hooks";
|
||||
}
|
||||
for my $dir ( @{ $hashref->{directories} } ) {
|
||||
grep( s/\//\\\//, @{ $dir->{repos} } );
|
||||
}
|
||||
|
||||
%conf = %$hashref;
|
||||
}
|
||||
|
||||
|
||||
sub curl_pipeline($handle) {
|
||||
add_callback( $handle, \&json_decode );
|
||||
add_callback( $handle, \&url_filter );
|
||||
add_callback( $handle, \&process_urls );
|
||||
add_callback( $handle, \&handle_repos );
|
||||
add_callback($handle, \&json_decode);
|
||||
add_callback($handle, \&url_filter);
|
||||
add_callback($handle, \&process_urls);
|
||||
|
||||
# add_callback($handle, \&dump);
|
||||
add_callback($handle, \&handle_repos);
|
||||
}
|
||||
|
||||
# ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░
|
||||
|
|
@ -231,32 +262,23 @@ read_conf();
|
|||
|
||||
my $last_handle = 0;
|
||||
|
||||
for my $i ( keys @{ $conf{lookups} } ) {
|
||||
my %lookup = %{ $conf{lookups}[$i] };
|
||||
chomp( $ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "" );
|
||||
for ( @{ $lookup{extra_headers} } ) {
|
||||
for my $i (keys @{$conf{lookups}}) {
|
||||
my %lookup = %{$conf{lookups}[$i]};
|
||||
chomp($ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "");
|
||||
for (@{$lookup{extra_headers}}) {
|
||||
$_ = `printf "$_"`;
|
||||
}
|
||||
if ( $lookup{targets} ) {
|
||||
for my $j ( keys @{ $lookup{targets} } ) {
|
||||
if ($lookup{targets}) {
|
||||
for my $j (keys @{$lookup{targets}}) {
|
||||
$last_handle++;
|
||||
$handles[$last_handle]{lookup} = $i;
|
||||
set_curl(
|
||||
$last_handle,
|
||||
"$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}",
|
||||
@{ $lookup{extra_headers} }
|
||||
);
|
||||
set_curl($last_handle, "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}", @{$lookup{extra_headers}});
|
||||
curl_pipeline($last_handle);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
$last_handle++;
|
||||
$handles[$last_handle]{lookup} = $i;
|
||||
set_curl(
|
||||
$last_handle,
|
||||
"$lookup{api_url}/$lookup{endpoint}",
|
||||
@{ $lookup{extra_headers} }
|
||||
);
|
||||
set_curl($last_handle, "$lookup{api_url}/$lookup{endpoint}", @{$lookup{extra_headers}});
|
||||
curl_pipeline($last_handle);
|
||||
}
|
||||
}
|
||||
|
|
@ -264,9 +286,9 @@ for my $i ( keys @{ $conf{lookups} } ) {
|
|||
exec_multicurl();
|
||||
|
||||
$last_handle++;
|
||||
add_callback( $last_handle, \&inject_conf_urls );
|
||||
add_callback( $last_handle, \&process_urls );
|
||||
add_callback( $last_handle, \&handle_repos );
|
||||
add_callback($last_handle, \&inject_conf_urls);
|
||||
add_callback($last_handle, \&process_urls);
|
||||
add_callback($last_handle, \&handle_repos);
|
||||
exec_callbacks($last_handle);
|
||||
|
||||
for my $message (@messages) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue