finish work on clonedev

.config/clonedev/config

@@ -1,78 +0,0 @@
-#!/bin/sh
-# TODO: Rewrite this in a proper language
-# shellcheck disable=2034
-set -a
-
-BASE="$XDG_DOCUMENTS_DIR/dev"
-
-# TOKEN has to be an API token (PAT) with
-# the following permissions (for the default configuration)
-# read:organization
-# read:repository
-# read:user
-TOKEN="$(secret-tool lookup gitea api_token)"
-# Store with `secret-tool store --label=clonedev gitea api_token`
-
-# URL hit by curl is constructed like this: API_URL/TARGET/ENDPOINT
-# TARGET is extracted from a space separated list stored in TARGETS
-API_URL="https://git.snaile.de/api/v1"
-TARGETS="orgs/snailed users/luca"
-ENDPOINT="repos"
-HEADERS="
-    -H 'accept: application/json'
-    -H 'Authorization: token $TOKEN'
-"
-
-# The output of this request are filtered using this jq command:
-FILTER=".[].ssh_url"
-
-# After filtering, the output passed to a git clone command with these flags:
-CLONE_FLAGS="--recursive -j8"
-
-# If the repo is listed in one of these variables, it is moved to
-# the dir in the variable name. The dirname must match this regex: "^DEVDIRS_([A-Za-z_-]*)(?==)"
-# The following config would result in the testrepo repo being placed like this:
-
-# testdir/testrepo
-# test_block would be blocked from getting cloned
-
-# DEVDIRS_testdir="
-#     testrepo
-#
-# BLOCKDIRS="test_block"
-
-DEVDIRS_desktop="
-    bootstrapper
-    dmenu-custom
-    dwm-custom
-    dwmblocks-custom
-    st-custom
-    tokyonight-icons
-    tokyonight-theme
-"
-
-DEVDIRS_server="
-    ansible-example
-    ansible-homelab
-    dotfiles-server
-    void-packages
-    void-packages-custom
-    qbittorrent-natpmp
-    xbps-builder
-    matrix-docker-ansible-deploy
-    server-resources
-"
-DEVDIRS_misc="
-    tidal-scraper
-"
-DEVDIRS_nvim="
-    taolf
-"
-
-BLOCK_UNSORTED=false
-BLOCK_REPOS="dotfiles pinentry-dmenu-custom"
-
-EXTRA_REPOS="
-    https://github.com/void-linux/void-packages
-    https://github.com/spantaleev/matrix-docker-ansible-deploy
-"

.config/clonedev/config.yml

@@ -1,28 +1,31 @@
-plugins:
+lookups:
   - name: "gitea_owned_repos"
     api_url: "https://git.snaile.de/api/v1"
+    endpoint: "repos"
+    url_type: "ssh_url"
+    token_cmd: "secret-tool lookup gitea api_token"
+    block_unsorted: 0
     targets: 
       - "orgs/snailed"
       - "users/luca"
-    endpoint: "repos"
-    token_cmd: "secret-tool lookup gitea api_token"
     extra_headers:
       - "Authorization: token ${TOKEN}"
+  - name: "github_starred_repos"
+    api_url: "https://api.github.com"
+    endpoint: "starred"
     url_type: "ssh_url"
-  # - name: "github_starred_repos"
+    token_cmd: "secret-tool lookup github api_token"
-  #   api_url: "https://api.github.com"
+    block_unsorted: 1
-  #   targets: 
+    targets: 
-  #     - "users/ssnailed"
+      - "users/ssnailed"
-  #   endpoint: "starred"
+    extra_headers:
-  #   url_type: "ssh_url"
+      - "Authorization: Bearer ${TOKEN}"
-  #   token_cmd: "secret-tool lookup github api_token"
+
-  #   extra_headers:
+extra_urls:
-  #     - "Authorization: Bearer ${TOKEN}"
+  # - "https://git.kernel.org/pub/scm/git/git.git"
-        
-repos:
-  - "https://git.kernel.org/pub/scm/git/git.git"
 
 clone_flags: "--recursive -j8"
+pull_flags: "--recurse-submodules=yes -j8"
 
 directories:
   - path: "${XDG_DOCUMENTS_DIR}/dev/desktop"
@@ -39,12 +42,12 @@ directories:
       - "snailed/ansible-example"
       - "snailed/ansible-homelab"
       - "snailed/dotfiles-server"
-      - "snailed/qbittorrent-natpmp"
+      - "snailed/qbittorrent-natpmp-docker"
+      - "snailed/papermc-docker"
       - "snailed/server-resources"
       - "snailed/void-packages-custom"
       - "snailed/xbps-builder"
       - "spantaleev/matrix-docker-ansible-deploy"
-      - "void-linux/void-packages"
   - path: "${XDG_DOCUMENTS_DIR}/dev/nvim"
     repos:
       - "snailed/taolf"
@@ -53,10 +56,13 @@ directories:
       - "snailed/tidal-scraper"
       - "b3nj5m1n/xdg-ninja"
       - "jnweiger/led-name-badge-ls32"
+  - path: "${XDG_DOCUMENTS_DIR}/dev"
+    repos:
+      - "void-linux/void-packages"
 
-block_unsorted: 0
 unsorted_directory: "${XDG_DOCUMENTS_DIR}/dev/unsorted"
 
 block_repos: 
+  - "luca/dotfiles"
   - "luca/pinentry-dmenu-custom"
   - "snailed/matrix-inventory"

.config/clonedev/hooks/clone/spantaleev:matrix-docker-ansible-deploy

@@ -0,0 +1,3 @@
+#!/bin/sh
+just roles
+git clone -- https://git.snaile.de/snailed/matrix-inventory inventory

.config/clonedev/hooks/install/matrix-docker-ansible-deploy

@@ -1,2 +0,0 @@
-#!/bin/sh
-submodule add -f -- https://git.snaile.de/snailed/matrix-inventory inventory

.config/clonedev/hooks/pull/spantaleev:matrix-docker-ansible-deploy

@@ -0,0 +1,3 @@
+#!/bin/sh
+just roles
+git -C inventory pull

.config/clonedev/hooks/update/matrix-docker-ansible-deploy

@@ -1,2 +0,0 @@
-#!/bin/sh
-just roles

.local/bin/clonedev

@@ -1,16 +1,5 @@
 #!/bin/perl
 
-#
-# TODO: detect existing repos and automatically modify config file accordingly
-# TODO: function to detect unclean working trees
-# TODO: Handle cloning
-# TODO: Handle pulling
-# TODO: Trigger hooks on clone/pull
-#
-
-print("This shit ain't done yet!\n");
-# exit;
-
 use strict;
 use warnings;
 use feature ("signatures");
@@ -18,17 +7,36 @@ use YAML();
 use JSON();
 use WWW::Curl::Easy;
 use WWW::Curl::Multi;
-use Data::Dump();
+use File::Path("make_path");
-use File::Path();
+
+# use Data::Dumper;
+# $Data::Dumper::Pair   = " : ";
+# $Data::Dumper::Indent = 2;
+
+# TODO: Async the git clones
+# TODO: Show hook/clone output in a prettier fashion (like docker buildx)
 
 use constant USERAGENT =>
   "User-Agent: MarxBot/4.2.0 (A script reading some information about repos)";
 
 my @handles;
+my @messages;
 my %conf;
 my $active_repos    = 0;
 my $active_requests = 0;
 
+sub info($message) {
+    print(`tput setaf 4; tput bold; echo '$message'; tput sgr 0`);
+}
+
+sub error($message) {
+    push( @messages, `tput setaf 1; tput bold; echo '$message'; tput sgr 0` );
+}
+
+sub warning($message) {
+    push( @messages, `tput setaf 3; tput bold; echo '$message'; tput sgr 0` );
+}
+
 sub set_curl( $handle, $url, @headers ) {
     $handles[$handle]{curl} = WWW::Curl::Easy->new;
     $handles[$handle]{curl}->setopt( CURLOPT_URL, $url );
@@ -45,6 +53,14 @@ sub add_callback( $handle, $callback ) {
     push( @{ $handles[$handle]{callbacks} }, $callback );
 }
 
+sub exec_callbacks($handle) {
+    if ( $handles[$handle]{callbacks} ) {
+        for my $callback ( @{ $handles[$handle]->{callbacks} } ) {
+            $callback->($handle);
+        }
+    }
+}
+
 sub exec_multicurl() {
     my $curlm = WWW::Curl::Multi->new;
     for my $handle (@handles) {
@@ -59,12 +75,7 @@ sub exec_multicurl() {
             while ( my ( $handle, $return_value ) = $curlm->info_read ) {
                 if ($handle) {
                     $active_requests--;
-                    if ( $handles[$handle]{callbacks} ) {
+                    exec_callbacks($handle);
-                        for my $callback ( @{ $handles[$handle]->{callbacks} } )
-                        {
-                            $callback->($handle);
-                        }
-                    }
                     delete $handles[$handle];
                 }
             }
@@ -76,59 +87,139 @@ sub json_decode($handle) {
     $handles[$handle]{memory} = JSON::decode_json( $handles[$handle]{memory} );
 }
 
-sub filter($handle) {
+sub url_filter($handle) {
     my @tmp;
-    my $plugin = $conf{plugins}[ $handles[$handle]->{plugin} ];
+    my $lookup = $conf{lookups}[ $handles[$handle]->{lookup} ];
-    foreach my $repo ( @{ $handles[$handle]{memory} } ) {
+    for my $repo ( @{ $handles[$handle]{memory} } ) {
-        if ( $repo->{ssh_url} ) {
+        if ( $repo->{ $lookup->{url_type} } ) {
-            push( @tmp, $repo->{ %$plugin{url_type} } );
+            push( @tmp, $repo->{ $lookup->{url_type} } );
+        }
+        else {
+            error(
+"Failed to extract $lookup->{url_type} while processing lookup: $lookup->{name}"
+            );
         }
     }
     $handles[$handle]{memory} = \@tmp;
 }
 
+sub inject_conf_urls($handle) {
+    for my $url ( @{ $conf{extra_urls} } ) {
+        push( @{ $handles[$handle]{memory} }, $url );
+    }
+}
+
 sub process_urls($handle) {
     my @tmp;
-    foreach my $url ( @{ $handles[$handle]{memory} } ) {
+    for my $url ( @{ $handles[$handle]{memory} } ) {
         my %repo;
         if ( $url =~
-            /^(.*[\/:](([a-zA-Z0-9-_.]+)\/([a-zA-Z0-9-_.]+?))(?:\.git)?\/?)$/ )
+/^((.*(?:@|:\/\/))[a-zA-Z0-9-_.]+(?:\/|:)(?:[a-zA-Z0-9-_.\/]+\/)?([a-zA-Z0-9-_.]+?)\/([a-zA-Z0-9-_.]+?)(?:\.git)?\/?)$/
+          )
         {
-            $repo{url}  = $1;
+            $repo{url}      = $1;
-            $repo{name} = $2;
+            $repo{owner}    = $3;
+            $repo{name}     = $4;
+            $repo{fullname} = "$3/$4";
+
+            if ( substr( $2, -1 ) eq "@" ) {
+                $repo{protocol} = "ssh";
+            }
+            else {
+                $repo{protocol} = $2;
+            }
         }
         else {
-            return;
+            error("Failed to parse url: $url");
+            next;
         }
-        continue if grep /^$repo{name}$/, $conf{block_repos};
+
         my $path;
-        foreach my $directory ( @{ $conf{directories} } ) {
+        for my $directory ( @{ $conf{directories} } ) {
-            foreach my $reponame ( @{ %$directory{repos} } ) {
+            for my $reponame ( @{ %$directory{repos} } ) {
-                if ( $reponame =~ $repo{name} ) {
+                if ( $reponame eq "$repo{fullname}" ) {
                     $path = %$directory{path};
                     last;
                 }
             }
         }
-        if ( !$path and !$conf{block_unsorted} ) {
+
-            $path = "\${XDG_DOCUMENTS_DIR}/unsorted_repos";
+        if ( !$handles[$handle]{lookup} ) {
-            $repo{path} = `printf $path`;
+            if ($path) {
-            push( @tmp, %repo );
+                $repo{path} = `printf $path/$repo{name}`;
+            }
+            else {
+                $repo{path} = `printf $conf{unsorted_directory}/$repo{name}`;
+            }
+
         }
+        elsif ( !$conf{lookups}[ $handles[$handle]{lookup} ]{block_unsorted}
+            && $repo{path} )
+        {
+            $repo{path} = `printf $conf{unsorted_directory}/$repo{name}`;
+        }
+        elsif ($path) {
+            $repo{path} = `printf $path/$repo{name}`;
+        }
+        else {
+            warning("Skipping $repo{fullname}");
+            next;
+        }
+
+        my $clone_hook = "$conf{hook_dir}/clone/$repo{owner}:$repo{name}";
+        my $pull_hook  = "$conf{hook_dir}/pull/$repo{owner}:$repo{name}";
+        ( -x $clone_hook )
+          and $repo{clone_hook} = "cd $repo{path} && $clone_hook";
+        ( -x $pull_hook ) and $repo{pull_hook} = "cd $repo{path} && $pull_hook";
+        push( @tmp, \%repo );
     }
     $handles[$handle]{memory} = \@tmp;
 }
+#
+# sub dump($handle) {
+#     print("------ Handle $handle ------\n");
+#     print Dumper( $handles[$handle]->{memory} );
+# }
 
-sub dump($handle) {
+sub folder_is_empty($directory) {
-    print("------ Handle $handle ------\n");
+    opendir( my $dh, $directory ) or return 1;
-    Data::Dump::dump( $handles[$handle]{memory} );
+    return scalar( grep { $_ ne "." && $_ ne ".." } readdir($dh) ) == 0;
 }
 
-# sub handle_repos($handle) {
+sub handle_repos($handle) {
-#     foreach my $repo (@{$handles[$handle]->{memory}}) {
+    for my $repo ( @{ $handles[$handle]->{memory} } ) {
-#         if (-d $repo{})
+        if ( folder_is_empty("$repo->{path}") ) {
-#     }
+            make_path( $repo->{path} );
-# }
+            info("Cloning $repo->{fullname}");
+            `git -C '$repo->{path}' clone $conf{clone_flags} '$repo->{url}' .`;
+            ( $? != 0 )
+              and error("Failed to clone $repo->{url} to $repo->{path}");
+            if ( $repo->{clone_hook} ) {
+                info("Running clone hook for $repo->{fullname}");
+                `$repo->{clone_hook}`;
+            }
+            ( $? != 0 )
+              and error("Failed to execute clone hook for $repo->{fullname}");
+        }
+        elsif ( !folder_is_empty("$repo->{path}/.git") ) {
+            info("Pulling $repo->{fullname} to $repo->{path}");
+            if (`git -C $repo->{path} status -z`) {
+                warn("$repo->{path} has an unclean tree.");
+            }
+            else {
+                `git -C $repo->{path} pull $conf{pull_flags}`;
+            }
+            ( $? != 0 )
+              and error("Failed to pull $repo->{url} to $repo->{path}");
+            if ( $repo->{pull_hook} ) {
+                info("Running pull hook for $repo->{fullname}");
+                `$repo->{pull_hook}`;
+            }
+            ( $? != 0 )
+              and error("Failed to execute pull hook for $repo->{fullname}");
+        }
+    }
+}
 
 sub read_conf() {
     my $configdir;
@@ -139,8 +230,11 @@ sub read_conf() {
         $configdir = "$ENV{HOME}/.config/clonedev";
     }
     open( my $cfg, '<', $configdir . "/config.yml" ) or die;
-    my ( $hashref, $arrayref, $string ) = YAML::Load( do { local $/; <$cfg> } );
+    my $hashref = YAML::Load( do { local $/; <$cfg> } );
     close($cfg);
+    if ( !$hashref->{hook_dir} ) {
+        $hashref->{hook_dir} = "$configdir/hooks";
+    }
     %conf = %$hashref;
 }
 
@@ -152,43 +246,37 @@ sub read_conf() {
 # █  ████  ██  ████  ██        ██  ███   █
 # ████████████████████████████████████████
 
-
 read_conf();
 
-# $last_handle++;
-# set_curl( $last_handle, "https://api.github.com/octocat" );
-# add_callback(
-#     $last_handle,
-#     sub ($handle) {
-#         print( $handles[$handle]{memory} );
-#     }
-# );
-#
 my $last_handle = 0;
-foreach my $i ( keys @{ $conf{plugins} } ) {
+
-    my %plugin = %{ $conf{plugins}[$i] };
+for my $i ( keys @{ $conf{lookups} } ) {
-    chomp(
+    my %lookup = %{ $conf{lookups}[$i] };
-        $ENV{TOKEN} =
+    chomp( $ENV{TOKEN} = $lookup{token_cmd} ? `$lookup{token_cmd}` : "" );
-          $plugin{token_cmd}
+    for ( $lookup{extra_headers} ) {
-        ? `$plugin{token_cmd}`
-        : ""
-    );
-    foreach ( @{ $plugin{extra_headers} } ) {
-        last if ( !$_ );
         $_ = `printf "$_"`;
     }
-    foreach my $j ( keys @{ $plugin{targets} } ) {
+    for my $j ( keys @{ $lookup{targets} } ) {
         $last_handle++;
-        $handles[$last_handle]{plugin} = $i;
+        $handles[$last_handle]{lookup} = $i;
-        print("$plugin{api_url}/$plugin{targets}[$j]/$plugin{endpoint}\n");
+        set_curl( $last_handle,
-        set_curl( $last_handle, "$plugin{api_url}/$plugin{targets}[$j]/$plugin{endpoint}",
+            "$lookup{api_url}/$lookup{targets}[$j]/$lookup{endpoint}",
-            $plugin{extra_headers} );
+            $lookup{extra_headers} );
-        add_callback( $last_handle, \&dump );
         add_callback( $last_handle, \&json_decode );
-        add_callback( $last_handle, \&filter );
+        add_callback( $last_handle, \&url_filter );
         add_callback( $last_handle, \&process_urls );
-        add_callback( $last_handle, \&dump );
+        add_callback( $last_handle, \&handle_repos );
     }
 }
 
 exec_multicurl();
+
+$last_handle++;
+add_callback( $last_handle, \&inject_conf_urls );
+add_callback( $last_handle, \&process_urls );
+add_callback( $last_handle, \&handle_repos );
+exec_callbacks($last_handle);
+
+for my $message (@messages) {
+    print($message);
+}