Merge pull request #105 from limanmys/go_self

Replikasyon Sistemi

.github/workflows/main.yml

@@ -99,7 +99,7 @@ jobs:
           Architecture: amd64
           Priority: important
           Description: Liman MYS
-          Depends: curl, gpg, zip, unzip, nginx, redis, php-redis, php7.3-fpm, php7.3-curl, php7.3, php7.3-sqlite3, php7.3-snmp, php7.3-mbstring, php7.3-xml, php7.3-zip, php7.3-posix, libnginx-mod-http-headers-more-filter, libssl1.1, supervisor, postgresql-13, php7.3-pgsql, pgloader, php7.3-bcmath
+          Depends: curl, gpg, zip, unzip, nginx, redis, php-redis, php7.3-fpm, php7.3-curl, php7.3, php7.3-sqlite3, php7.3-snmp, php7.3-mbstring, php7.3-xml, php7.3-zip, php7.3-posix, libnginx-mod-http-headers-more-filter, libssl1.1, supervisor, postgresql-13, php7.3-pgsql, pgloader, php7.3-bcmath, rsync, dnsutils, php7.3-ldap, php-smbclient, krb5-user, php-ssh2, smbclient, novnc, python3.7, python3-jinja2, python3-requests, python3-crypto, python3-paramiko, python3-tornado
           """ > DEBIAN/control
           cd ../
           dpkg-deb -Zgzip --build package

app/Console/Kernel.php

@@ -40,6 +40,14 @@ class Kernel extends ConsoleKernel
             ->dailyAt("23:59")
             ->name('Token Cleanup');
 
+        // Sync files.
+        $schedule
+            ->call(function () {
+                syncFiles();
+            })
+            ->everyFiveMinutes()
+            ->name('Sync extensions');
+
         // Run Health Check every hour.
         $schedule
             ->call(function () {

app/Http/Controllers/Extension/MainController.php

@@ -14,7 +14,6 @@ use Illuminate\Support\Str;
 use Illuminate\View\View;
 use Illuminate\Support\Facades\Validator;
 use App\Jobs\ExtensionUpdaterJob;
-use App\Jobs\ExtensionDependenciesJob;
 use Illuminate\Contracts\Bus\Dispatcher;
 use App\Models\AdminNotification;
 
@@ -229,27 +228,7 @@ class MainController extends Controller
         $new->save();
         
         if (array_key_exists("dependencies", $json) && $json["dependencies"] != "") {
-            $job = (new ExtensionDependenciesJob(
-                $new,
-                $json["dependencies"]
-            ))->onQueue('system_updater');
-    
-            // Dispatch job right away.
-            $job_id = app(Dispatcher::class)->dispatch($job);
-
-            AdminNotification::create([
-                "title" =>
-                    $new->display_name . " eklentisinin bağımlılıkları yükleniyor!",
-                "type" => "",
-                "message" =>
-                    $new->display_name .
-                    " eklentisinin bağımlılıkları yükleniyor, bu süre içerisinde eklentiyi kullanamazsınız.",
-                "level" => 3,
-            ]);
-            $new->update([
-                "status" == "0"
-            ]);
-            $new->save();
+            rootSystem()->installPackages($json["dependencies"]);
         }
 
         $system = rootSystem();

app/Http/Controllers/Extension/OneController.php

@@ -15,7 +15,6 @@ use Illuminate\Support\Str;
 use mervick\aesEverywhere\AES256;
 use GuzzleHttp\Client;
 use App\Models\Token;
-use App\Jobs\ExtensionDependenciesJob;
 use Illuminate\Contracts\Bus\Dispatcher;
 use App\Models\AdminNotification;
 
@@ -218,11 +217,6 @@ class OneController extends Controller
 
     public function forceDepInstall()
     {
-        $flag =extension()->update([
-            "status" => "0"
-        ]);
-        
-
         $file = file_get_contents("/liman/extensions/" .strtolower(extension()->name) . "/db.json");
         $json = json_decode($file,true);
         if(json_last_error() != JSON_ERROR_NONE){
@@ -230,30 +224,7 @@ class OneController extends Controller
         }
 
         if (array_key_exists("dependencies",$json) && $json["dependencies"] != ""){
-            $job = (new ExtensionDependenciesJob(
-                extension(),
-                $json["dependencies"]
-            ))->onQueue('system_updater');
-    
-            // Dispatch job right away.
-            $job_id = app(Dispatcher::class)->dispatch($job);$job = (new ExtensionDependenciesJob(
-                extension(),
-                $json["dependencies"]
-            ))->onQueue('system_updater');
-    
-            // Dispatch job right away.
-            $job_id = app(Dispatcher::class)->dispatch($job);
-
-            AdminNotification::create([
-                "title" =>
-                    extension()->display_name . " eklentisinin bağımlılıkları yükleniyor!",
-                "type" => "",
-                "message" =>
-                    extension()->display_name .
-                    " eklentisinin bağımlılıkları yükleniyor, bu süre içerisinde eklentiyi kullanamazsınız.",
-                "level" => 3,
-            ]);
-
+            rootSystem()->installPackages($json["dependencies"]);
             return respond("İşlem başlatıldı!");
         }else{
             return respond("Bu eklentinin hiçbir bağımlılığı yok!",201);

app/Http/Helpers.php

@@ -3,9 +3,12 @@
 use App\Models\AdminNotification;
 use App\Models\Extension;
 use App\Models\Notification;
+use App\Models\SystemSettings;
 use App\Models\Permission;
 use App\Models\Server;
 use App\Models\Certificate;
+use App\Models\Liman;
+use App\Models\Module;
 use App\User;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Query\Builder;
@@ -19,6 +22,120 @@ use Illuminate\Support\Str;
 use Jenssegers\Blade\Blade;
 use App\System\Helper;
 use mervick\aesEverywhere\AES256;
+use phpseclib\Crypt\RSA;
+
+if (!function_exists('updateSystemSettings')) {
+    function updateSystemSettings()
+    {
+        SystemSettings::updateOrCreate(
+            ['key' => 'APP_KEY'],
+            ['data' => env('APP_KEY')]
+        );
+
+        SystemSettings::updateOrCreate(
+            ['key' => 'LIMAN_RESTRICTED'],
+            ['data' => env('LIMAN_RESTRICTED',false)]
+        );
+
+        SystemSettings::updateOrCreate(
+            ['key' => 'SSL_PUBLIC_KEY'],
+            ['data' => file_get_contents("/liman/certs/liman.crt")]
+        );
+
+        SystemSettings::updateOrCreate(
+            ['key' => 'SSL_PRIVATE_KEY'],
+            ['data' => file_get_contents("/liman/certs/liman.key")]
+        );
+        $sshPublic = SystemSettings::where([
+            "key" => "SSH_PUBLIC"
+        ])->first();
+        if(!$sshPublic){
+            $rsa = new RSA();
+            $rsa->setPublicKeyFormat(RSA::PUBLIC_FORMAT_OPENSSH);
+            extract($rsa->createKey());
+            `mkdir -p /home/liman/.ssh`;
+            file_put_contents("/home/liman/.ssh/authorized_keys",$publickey);
+            file_put_contents("/home/liman/.ssh/liman_pub",$publickey);
+            file_put_contents("/home/liman/.ssh/liman_priv",$privatekey);
+            
+            chmod("/home/liman/.ssh/liman_pub",0600);
+            chmod("/home/liman/.ssh/liman_priv",0600);
+
+            SystemSettings::create([
+                "key" => "SSH_PUBLIC",
+                "data" => $publickey
+            ]);
+
+            SystemSettings::updateOrCreate(
+                ['key' => 'SSH_PRIVATE_KEY'],
+                ['data' => $privatekey]
+            );
+        }
+    }
+}
+
+if (!function_exists('receiveSystemSettings')) {
+    function receiveSystemSettings()
+    {
+        $app_key = SystemSettings::where([
+            "key" => "APP_KEY"
+        ])->first();
+
+        if($app_key){
+            setEnv([
+                "APP_KEY" => $app_key->data
+            ]);
+        }
+        
+        $restricted = SystemSettings::where([
+            "key" => "LIMAN_RESTRICTED"
+        ])->first();
+
+        if($restricted){
+            setEnv([
+                "LIMAN_RESTRICTED" => $restricted->data
+            ]);
+        }
+
+        $public_key = SystemSettings::where([
+            "key" => "SSL_PUBLIC_KEY"
+        ])->first();
+
+        if($public_key){
+            file_put_contents("/liman/certs/liman.crt",$public_key->data);
+        }
+
+        $private_key = SystemSettings::where([
+            "key" => "SSL_PRIVATE_KEY"
+        ])->first();
+
+        if($private_key){
+            file_put_contents("/liman/certs/liman.key",$private_key->data);
+        }
+
+        $sshPublic = SystemSettings::where([
+            "key" => "SSH_PUBLIC"
+        ])->first();
+
+        if ($sshPublic) {
+            `mkdir -p /home/liman/.ssh`;
+            file_put_contents("/home/liman/.ssh/authorized_keys",$sshPublic->data);
+            file_put_contents("/home/liman/.ssh/liman_pub",$sshPublic->data);            
+            chmod("/home/liman/.ssh/liman_pub",0600);
+        }
+
+        $sshPrivate = SystemSettings::where([
+            "key" => "SSH_PRIVATE_KEY"
+        ])->first();
+
+        if ($sshPrivate) {
+            `mkdir -p /home/liman/.ssh`;
+            file_put_contents("/home/liman/.ssh/liman_priv",$sshPrivate->data);            
+            chmod("/home/liman/.ssh/liman_priv",0600);
+        }
+
+    }
+}
 
 if (!function_exists('respond')) {
     /**
@@ -49,6 +166,69 @@ if (!function_exists('respond')) {
     }
 }
 
+if (!function_exists('syncFiles')) {
+    function syncFiles()
+    {
+        $masterIp = env('LIMAN_MASTER_IP');
+
+        if($masterIp == ""){
+            $firstLiman = Liman::first();
+            $masterIp = $firstLiman->last_ip;
+        }
+
+        shell_exec("rsync -Pav -e \"ssh -i /home/liman/.ssh/liman_priv -o 'StrictHostKeyChecking no'\" liman@" . $masterIp . ":/liman/extensions/ /liman/extensions/");
+        shell_exec("rsync -Pav -e \"ssh -i /home/liman/.ssh/liman_priv -o 'StrictHostKeyChecking no'\" --exclude 'service.key' liman@" . $masterIp . ":/liman/keys/ /liman/keys/");
+        shell_exec("rsync -Pav -e \"ssh -i /home/liman/.ssh/liman_priv -o 'StrictHostKeyChecking no'\" liman@" . $masterIp . ":/liman/modules/ /liman/modules/");
+        
+        $root = rootSystem();
+        $extensions = Extension::all();
+        $names =[];
+
+        foreach($extensions as $extension){
+            array_push($names,strtolower($extension->name));
+            $root->userAdd($extension->id);
+            $root->fixExtensionPermissions($extension->id,$extension->name);
+            $json = getExtensionJson($extension->name);
+            if(array_key_exists("dependencies",$json) && $json["dependencies"] != ""){
+                $root->installPackages($json["dependencies"]);
+            }
+        }
+
+        $scan = scandir('/liman/extensions/');
+
+        foreach($scan as $a){
+            if(substr($a,0,1) == ".") {
+                continue;
+            }
+            if(!in_array($a,$names)){
+                `rm -rf /liman/extensions/$a`;
+            }
+        }
+
+        $dns = SystemSettings::where([
+            "key" => "SYSTEM_DNS"
+        ])->first();
+        if($dns){
+            $json = json_decode($dns->data);
+            $root->dnsUpdate($json[0],$json[1],$json[2]);
+        }
+
+        $certificates = SystemSettings::where([
+            "key" => "SYSTEM_CERTIFICATES"
+        ])->first();
+        if($certificates){
+            $json = json_decode($certificates->data,true);
+            foreach($json as $cert){
+                if(is_file("/usr/local/share/ca-certificates/" . $cert["targetName"] . ".crt")){
+                    continue;
+                }
+                $root->addCertificate($cert["certificate"],$cert["targetName"]);
+            }
+        }
+    }
+}
+
+
 if (!function_exists('ip_in_range')) {
     function ip_in_range($ip, $range)
     {
@@ -782,7 +962,6 @@ if (!function_exists('setEnv')) {
             return false;
         }
         shell_exec('php /liman/server/artisan config:clear');
-        shell_exec("sed -i -e :a -e '/^\n*$/{\$d;N;ba' -e '}' /liman/server/.env");
         return true;
     }
 }

app/Models/Liman.php

+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+
+class Liman extends Model
+{
+    use UsesUuid;
+
+    protected $table = "limans";
+
+    protected $fillable = [
+        "machine_id" ,"last_ip"
+    ];
+}

app/Models/Replication.php

+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+
+class Replication extends Model
+{
+    use UsesUuid;
+}

app/Models/SystemSettings.php

+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+
+class SystemSettings extends Model
+{
+    use UsesUuid;
+
+    protected $fillable = [
+        "key", "data"
+    ];
+}

app/System/Helper.php

@@ -2,6 +2,7 @@
 
 namespace App\System;
 
+use App\Models\SystemSettings;
 use GuzzleHttp\Client;
 
 class Helper {
@@ -60,11 +61,58 @@ class Helper {
         }catch(\Exception $e){
             return false;
         }
+
+        SystemSettings::updateOrCreate(
+            ['key' => 'SYSTEM_DNS'],
+            ['data' => json_encode([
+                $server1, $server2, $server3
+            ])]
+        );
+
         return true;
     }
 
     public function addCertificate($tmpPath, $targetName)
     {
+        $contents = $tmpPath;
+        if(is_file($tmpPath)){
+            $contents = file_get_contents($tmpPath);
+        }else{
+            $tmpPath = "/tmp/" . str_random(16);
+            file_put_contents($tmpPath,$contents);
+        }
+        $arr = [
+            "certificate" => $contents,
+            "targetName" => $targetName
+        ];
+        
+        $current = SystemSettings::where("key", "SYSTEM_CERTIFICATES")->first();
+
+        if ($current) {
+            $foo = json_decode($current->data, true);
+            $flag = true;
+            for ($i = 0; $i < count($foo); $i++) {
+                if ($foo[$i]["targetName"] == $targetName) {
+                    $foo[$i]["certificate"] = $arr["certificate"];
+                    $flag = false;
+                    break;
+                }
+            }
+            
+            if ($flag) {
+                array_push($foo, $arr);
+            }
+            
+            $current->update([
+                "data" => json_encode($foo)
+            ]);
+        } else {
+            SystemSettings::create([
+                "key" => "SYSTEM_CERTIFICATES",
+                "data" => json_encode([$arr])
+            ]);
+        }
+
         try{
             $this->client->get('/certificateAdd',[
                 'query' => [
@@ -81,6 +129,26 @@ class Helper {
 
     public function removeCertificate($targetName)
     {
+        $arr = [
+            "targetName" => $targetName
+        ];
+        
+        $current = SystemSettings::where("key", "SYSTEM_CERTIFICATES")->first();
+
+        if ($current) {
+            $foo = json_decode($current->data, true);
+            for ($i = 0; $i < count($foo); $i++) {
+                if ($foo[$i]["targetName"] == $targetName) {
+                    unset($foo[$i]);
+                    $foo = array_values($foo);
+                    break;
+                }
+            }
+            $current->update([
+                "data" => $foo
+            ]);
+        }
+
         try{
             $this->client->get('/certificateRemove',[
                 'query' => [
@@ -110,6 +178,21 @@ class Helper {
         return true;
     }
 
+    public function installPackages($packages)
+    {
+        try{
+            $this->client->get('/installPackages',[
+                'query' => [
+                    'liman_token' => $this->authKey,
+                    'packages' => $packages,
+                ]
+            ]);
+        }catch(\Exception $e){
+            return false;
+        }
+        return true;
+    }
+
     public function runCommand($command)
     {
         try{

composer.json

@@ -7,11 +7,9 @@
     "require": {
         "php": "^7.3",
         "ext-json": "*",
-        "ext-ldap": "*",
         "ext-openssl": "*",
         "ext-posix": "*",
         "ext-snmp": "*",
-        "ext-ssh2": "*",
         "ext-xml": "*",
         "ext-zip": "*",
         "ankitpokhrel/tus-php": "^1.0",

database/migrations/2020_11_30_184347_create_system_settings_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateSystemSettingsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('system_settings', function (Blueprint $table) {
+            $table->uuid('id');
+            $table->string('key');
+            $table->string('data',10485760);
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('system_settings');
+    }
+}

database/migrations/2020_11_30_184609_create_limans_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateLimansTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('limans', function (Blueprint $table) {
+            $table->uuid('id');
+            $table->string('machine_id');
+            $table->string('last_ip');
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('limans');
+    }
+}

database/migrations/2020_11_30_184635_create_replications_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateReplicationsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('replications', function (Blueprint $table) {
+            $table->uuid('id');
+            $table->uuid('liman_id');
+            $table->string('key');
+            $table->integer('status')->default(0);
+            $table->string('output',9999);
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('replications');
+    }
+}

routes/console.php

@@ -3,6 +3,8 @@
 use App\User;
 use App\Models\Module;
 use App\Models\AdminNotification;
+use App\Models\Liman;
+use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\Hash;
 
 Artisan::command('administrator', function () {
@@ -148,3 +150,45 @@ Artisan::command('module:remove {module_name}', function ($module_name) {
         $this->error("Modul silinemedi.$flag");
     }
 })->describe("Module remove");
+
+Artisan::command('register_liman', function () {
+    Liman::updateOrCreate([
+        "machine_id" => getLimanId()
+    ],[
+        "last_ip" => env("LIMAN_IP",trim(`hostname -I`)),
+    ]);
+})->describe("Module remove");
+
+Artisan::command('update_settings', function () {
+    updateSystemSettings();
+})->describe("Update the system settings");
+
+Artisan::command('receive_settings', function () {
+    receiveSystemSettings();
+})->describe("Receive the system settings");
+
+Artisan::command('receive_settings', function () {
+    receiveSystemSettings();
+})->describe("Receive the system settings");
+
+
+Artisan::command('sync_core', function () {
+    if (trim(`id -u`) != "0") {
+        $this->error("Bu komutu root olarak çalışmalısınız!");
+        return;
+    }
+   
+    receiveSystemSettings();
+
+    `
+        systemctl restart nginx;
+        systemctl restart liman-render;
+        systemctl restart liman-system;
+        systemctl restart liman-socket;
+    `;
+})->describe("Sync core files.");
+
+
+Artisan::command('sync_safe', function () {
+    syncFiles();
+})->describe("Sync safe files without restarting");
\ No newline at end of file