Blender-Pipeline/network/http_client.py

# (c) J.Y.Amihud 2023
# GPL-3 or any later version

import os
import json
import time
import fnmatch
import threading
import hashlib
import urllib.request
import urllib.parse
import subprocess


# GTK module ( Graphical interface
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from gi.repository import GLib
from gi.repository import Gdk
import cairo

# Own modules
from settings import settings
from settings import talk
from project_manager import pm_project

#UI modules
from UI import UI_elements
from UI import UI_color

from studio import story
from studio import analytics

# This is a http client. To work on projects remotely.

def csize(x):

    x = float(x)

    l = ["B","KB", "MB", "GB", "TB"]

    for i in range(5):
        if x > 1024:
            x = x / 1024
        else:
            return str(round(x, 2))+" "+l[i]
    return str(round(x, 2))+" "+l[i]

def go(win, website, path):

    url = website+path
    print("url", url)
    response = urllib.request.urlopen(url)
    data = response.read()
    text = data.decode("utf-8")
    data = json.loads(text)
    
    return data

def down(win, website, filename, fsize=100000):

    # If it is a blend file, we are in trouble.

    # if filename.endswith(".blend"):
    #     dp = go(win, website, "/blend"+filename)
    #     dplen = len(dp.get("files", {}))
    #     for f in dp.get("files", {}):
    #         try:
    #             hashis = hashlib.md5(open(win.project+f,'rb').read()).hexdigest()
    #         except:
    #             hashis = ""
    #         if hashis != dp.get("files", {}).get(f, {}).get("md5"):
    #             win.current["http-server"]["message"] = f[f.rfind("/")+1:]
    #             filesize = dp.get("files", {}).get(f, {}).get("filesize")
    #             print("trying to get:", f)
    #             pp = down(win, website, f, filesize)
    
    # Making folder for the files
    
    try:
        os.makedirs(win.project+filename[:filename.rfind("/")+1])
    except:
        pass

    # downloading the file
    
    url = website+"/download"+filename
    response = urllib.request.urlopen(url)
    savef = open(win.project+filename, "wb")



    
    sofar = 0
    chunk = response.read(8192)
    while chunk:
        savef.write(chunk)
        chunk = response.read(8192)
        sofar = sofar + 8192
        win.current["http-server"]["message"] = filename[filename.rfind("/")+1:]
        win.current["http-server"]["fileprog"] = sofar / fsize

    savef.close()

def download_missing_changed(win, cur):

    # This function downloads files from remote server using the
    # remote-folder-data data.

    website = win.analytics["remote-server-url"]

    for t in ["missing", "changed"]:
        while win.current["remote-folder-data"][cur][t]: # IDK WHAT I'M DOING!!!
            try:
                for f in win.current["remote-folder-data"][cur][t]:
                    fdata = win.current["remote-folder-data"][cur][t][f]
                    if fdata.get("to_download", True):
                        print("Downloading", f)
                        down(win, website, f, fdata.get("filesize", 10000))
                        del win.current["remote-folder-data"][cur][t][f]
            except:
                pass

    win.current["remote-folder-data"][cur]["downloading"] = False
                
def get_folder_info(win, folders, cur):

    # This function gets an information of what files should be changed. And outputs
    # the found data into the the global win thing, so it could accessed later.

    # Making sure we have the infrastructure.

    print("Getting Update Info About Folders")
    website = win.analytics["remote-server-url"]
    
    
    if "remote-folder-data" not in win.current:
        win.current["remote-folder-data"] = {}

    if cur not in win.current["remote-folder-data"]:
        win.current["remote-folder-data"][cur] = {"missing":{},
                                                  "changed":{}}

    def sort_file(filename, files, f):
        filedata = files.get(f, {})
        filedata["finished"] = False 
        try:
            hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
        except:
            hashis = ""
        if not os.path.exists(win.project+filename):
            win.current["remote-folder-data"][cur]["missing"][filename] = filedata
        elif hashis != filedata.get("md5"):
            filedata["to_download"] = True
            win.current["remote-folder-data"][cur]["changed"][filename] = filedata
            
        
    # Part one: Getting all the data about the files.
        
    for foldername in folders:

        try:
            answer = go(win, website, "/list"+foldername)
            files = list(answer.get("files", {}).keys())
            for f in fnmatch.filter(files, folders.get(foldername, "*")):

                # Now we need to figure out whether we have the file
                # , whether we don't even have it ,
                # or whether we have a different version.

                fullfilename = foldername+"/"+f
                sort_file(fullfilename, answer.get("files", {}), f)

                


        except Exception as e:
            error = "Error: "+ str(e)
            print(error)


    # Part two: Unwrapping all the blend dependancies data.

    # Here we are going to store blend file that were checked already.
    win.current["remote-folder-data"][cur]["blend_checked"] = []

    # A function checking if we still have some unchecked .blends
    def not_checked():
        for t in ["missing", "changed"]:
            for i in win.current["remote-folder-data"][cur][t]:
                if i not in win.current["remote-folder-data"][cur]["blend_checked"] and i.endswith(".blend"):
                    return i
        return False

    nc = not_checked()
    
    while nc:
        dp = go(win, website, "/blend"+nc)
        
        for f in dp.get("files", {}):
            try:
                sort_file(f,
                          dp.get("files", {}),
                          f[f.rfind("/")+1:])
            except Exception as e:
                print(e)
            
        win.current["remote-folder-data"][cur]["blend_checked"].append(nc)
            
        nc = not_checked()

    for t in ["missing", "changed"]:
        print(t+":")
        for i in win.current["remote-folder-data"][cur][t]:
            print("    "+i)

def get_folders(win):

    # This function will get specified subfolders
    win.current["http-server"]["message"] = "Getting Remote Folders"
    
    folder = win.current["http-server"]["args"]
    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]

    try:
        answer = go(win, website, "/list"+folder)
        for n, i in enumerate(answer.get("folders", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("folders", []))
            win.current["http-server"]["message"] = i
            try:

                os.makedirs(win.project+folder+"/"+i)
            except:
                pass
                
    except Exception as e:
        error = "Error: "+ str(e)
        print(error)
        win.current["http-server"]["message"] = error
        time.sleep(1)
        
    
    # Ending the process
    
    win.current["calls"][call]["var"] = False

def get_asset_files(win):

    # This function will get specified subfolders
    win.current["http-server"]["message"] = "Getting / Updating Files"
    
    folder = win.current["http-server"]["args"]
    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]

    folder = folder.replace("//", "/")
    if folder.endswith("/"):
        folder = folder[:-1]
    
    try:
        answer = go(win, website, "/list"+folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            filename = folder+"/"+i            
            
            try:
                hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
            except:
                hashis = ""

            if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                filesize = answer.get("files", {}).get(i, {}).get("filesize")
                down(win, website, filename, filesize)


        win.current["http-server"]["message"] = "Getting Thumbnail"
        other_folder = folder+"/renders"
        print("otherfolder 1", other_folder)
        answer = go(win, website, "/list"+other_folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            if i.startswith("Preview"):

                print("getting", i)
                
                filename = other_folder+"/"+i            

                try:
                    hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
                except:
                    hashis = ""

                if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                    filesize = answer.get("files", {}).get(i, {}).get("filesize")
                    down(win, website, filename, filesize)

        win.current["http-server"]["message"] = "Getting Asset Blend"
        other_folder = folder[:folder.rfind("/")+1].replace("/dev/", "/ast/")
        print("otherfolder 2", other_folder, folder)
        answer = go(win, website, "/list"+other_folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            if i.startswith(folder[folder.rfind("/")+1:]):

                print("getting", i)
                
                filename = other_folder+"/"+i            

                try:
                    hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
                except:
                    hashis = ""

                if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                    filesize = answer.get("files", {}).get(i, {}).get("filesize")
                    down(win, website, filename, filesize)
            
                
    except Exception as e:
        error = "Error: "+ str(e)
        print(error)
        win.current["http-server"]["message"] = error
        time.sleep(1)

    
    # Ending the process
    
    win.current["calls"][call]["var"] = False
    
def get_files(win):

    # This function will get specified subfolders
    win.current["http-server"]["message"] = "Getting / Updating Files"
    
    folder = win.current["http-server"]["args"]
    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]

    try:
        answer = go(win, website, "/list"+folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            filename = folder+"/"+i            
            
            try:
                hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
            except:
                print("missing", filename)
                hashis = ""

            if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                filesize = answer.get("files", {}).get(i, {}).get("filesize")
                down(win, website, filename, filesize)

            
                
    except Exception as e:
        error = "Error: "+ str(e)
        print(error)
        win.current["http-server"]["message"] = error
        time.sleep(1)
    
    # Ending the process
    
    win.current["calls"][call]["var"] = False
    
def get_essentials(win):

    # This function will get essential parts of the project.

    # Essentials are: Story, Analitycs, Project.progress  and Banner Image.

    essentials = [
        "/pln/story.vcss",
        "/set/analytics.json",
        "/set/project.progress",
        "/set/banner.png"]

    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]
    
    for n, i in enumerate(essentials):
        win.current["http-server"]["progress"] = n/len(essentials)
        win.current["http-server"]["message"] = i

        try:
            down(win, website, i)
        except Exception as e:
            error = "Error: "+ str(e)
            print(error)
            win.current["http-server"]["message"] = error
            time.sleep(1)
            
            
    win.story        = story.load(win.project)
    win.analytics    = analytics.load(win.project)

    win.analytics["remote-server-url"] = website
    win.analytics["from-remote-server"] = True
    analytics.save(win.project, win.analytics)
            
    # Ending the process
    
    win.current["calls"][call]["var"] = False
    
    

def layer(win, call):
    
    # Making the layer
    surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, win.current['w'],
                                                      win.current['h'])
    layer = cairo.Context(surface)
    
    
    #text setting
    layer.select_font_face("Monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
    
    UI_color.set(layer, win, "dark_overdrop")
    layer.rectangle(
        0,
        0,
        win.current["w"],
        win.current["h"],
        )
    layer.fill()
    
    # So it's going to be like a little window in the center of the VCStudio
    # with a simple UI. Probably like 2 things. Folder and a projectname.
    
    UI_color.set(layer, win, "node_background")
    UI_elements.roundrect(layer, win, 
        win.current["w"]/2-250,
        win.current["h"]/2-60,
        500,
        150,
        10)
    
    # Title of the operation. Incase the user forgot. 
    UI_elements.text(layer, win, "scan_project_title",
        win.current["w"]/2-250,
        win.current["h"]/2-35,
        500,
        30,
        10,
        fill=False,
        centered=True,
        editable=False)
    
    win.text["scan_project_title"]["text"] = win.current["http-server"]["message"]


    # File Progressbar
    UI_color.set(layer, win, "progress_background")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+10,
                          400,
                          20,
                          10,
                          tip="Hello")

    UI_color.set(layer, win, "progress_active")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+10,
                          400*max(0, min(1, win.current["http-server"]["fileprog"])),
                          20,
                          10,)

    
    # Progressbar
    UI_color.set(layer, win, "progress_background")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+40,
                          400,
                          20,
                          10,
                          tip="Hello")

    UI_color.set(layer, win, "progress_active")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+40,
                          400*max(0, min(1, win.current["http-server"]["progress"])),
                          20,
                          10,)
    
    return surface

def prompt_layer(win, call):
    
    # Making the layer
    surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, win.current['w'],
                                                      win.current['h'])
    layer = cairo.Context(surface)
    
    
    #text setting
    layer.select_font_face("Monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
    
    UI_color.set(layer, win, "dark_overdrop")
    layer.rectangle(
        0,
        0,
        win.current["w"],
        win.current["h"],
        )
    layer.fill()
    
    # So it's going to be like a little window in the center of the VCStudio
    # with a simple UI. Probably like 2 things. Folder and a projectname.
    
    UI_color.set(layer, win, "node_background")
    UI_elements.roundrect(layer, win, 
        win.current["w"]/2-350,
        100,
        700,
        win.current["h"]-200,
        10)
    
    

    cur = win.current["remote-folder-data"]["prompt"]


    if "downloading" not in win.current["remote-folder-data"][cur]:
        win.current["remote-folder-data"][cur]["downloading"] = False
    
    if not win.current["remote-folder-data"][cur]["downloading"]:


        # Exit button
        def do():
            win.current["calls"][call]["var"] = False


        UI_elements.roundrect(layer, win, 
            win.current["w"]/2+310,
            win.current["h"]-140,
            40,
            40,
            10,
            button=do,
            icon="cancel",
            tip=talk.text("cancel"))

        # Download button
        def do():
            win.current["remote-folder-data"][cur]["downloading"] = True

            start_downaloding = threading.Thread(target=download_missing_changed,
                                             args=(win, cur, ))
            start_downaloding.setDaemon(True)
            start_downaloding.start()


        UI_elements.roundrect(layer, win, 
            win.current["w"]/2+270,
            win.current["h"]-140,
            40,
            40,
            10,
            button=do,
            icon="download",
            tip=talk.text("DownloadRemoteServer"))

    else:

        # Progress bar

        UI_color.set(layer, win, "progress_background")
        UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                              win.current["h"]-140,
                          400,
                          20,
                          10,
                          tip="Hello")

        UI_color.set(layer, win, "progress_active")
        UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                              win.current["h"]-140,
                          400*max(0, min(1, win.current["http-server"]["fileprog"])),
                          20,
                          10,)

        UI_color.set(layer, win, "text_normal")
        layer.set_font_size(15)
        layer.move_to(win.current["w"]/2-180,
                      win.current["h"]-145)
        layer.show_text(win.current["http-server"]["message"])
        
    # Clipping everything
    UI_elements.roundrect(layer, win, 
        win.current["w"]/2-350,
        100,
        700,
        win.current["h"]-260,
        10,
        fill=False)
    layer.clip()
    
    clip = [
        win.current["w"]/2-350,
        100,
        700,
        win.current["h"]-260]
    
    # Setting up the scroll
    if "http-prompt" not in win.scroll:
        win.scroll["http-prompt"] = 0

    
        
    current_Y = 0

    # Missing files

    if "missing_active" not in win.current["remote-folder-data"][cur]:
        win.current["remote-folder-data"][cur]["missing_active"] = False

    

    icon = "closed"
    if win.current["remote-folder-data"][cur]["missing_active"]:
        icon = "open"
        
    def do():
        win.current["remote-folder-data"][cur]["missing_active"] = not win.current["remote-folder-data"][cur]["missing_active"]
    
        
    UI_elements.roundrect(layer, win, 
                          win.current["w"]/2-340,
                          110 + current_Y + win.scroll["http-prompt"],
                          680,
                          40,
                          10,
                          button=do,
                          icon=icon,
                          tip=talk.text("MissingFiles"))

    UI_color.set(layer, win, "text_normal")
    layer.set_font_size(20)
    layer.move_to(win.current["w"]/2-290,
                  current_Y + win.scroll["http-prompt"] + 135)
    layer.show_text(talk.text("MissingFiles")+" ( "+str(len(win.current["remote-folder-data"][cur]["missing"]))+" ) ")
    
    current_Y += 50
            
    if win.current["remote-folder-data"][cur]["missing_active"]:
        for f in win.current["remote-folder-data"][cur]["missing"]:

            fdata = win.current["remote-folder-data"][cur]["missing"][f]
            
            if fdata.get("finished"):
                continue
            
            UI_color.set(layer, win, "text_normal")
            layer.set_font_size(20)
            layer.move_to(win.current["w"]/2-270,
                          current_Y + win.scroll["http-prompt"] + 135)
            layer.show_text(f)

            current_Y += 50


    # Updated files

    if "changed_active" not in win.current["remote-folder-data"][cur]:
        win.current["remote-folder-data"][cur]["changed_active"] = True

    

    icon = "closed"
    if win.current["remote-folder-data"][cur]["changed_active"]:
        icon = "open"
        
    def do():
        win.current["remote-folder-data"][cur]["changed_active"] = not win.current["remote-folder-data"][cur]["changed_active"]
    
        
    UI_elements.roundrect(layer, win, 
                          win.current["w"]/2-340,
                          110 + current_Y + win.scroll["http-prompt"],
                          680,
                          40,
                          10,
                          button=do,
                          icon=icon,
                          tip=talk.text("ChangedFiles"))

    UI_color.set(layer, win, "text_normal")
    layer.set_font_size(20)
    layer.move_to(win.current["w"]/2-290,
                  current_Y + win.scroll["http-prompt"] + 135)
    layer.show_text(talk.text("ChangedFiles")+" ( "+str(len(win.current["remote-folder-data"][cur]["changed"]))+" ) ")
    
    current_Y += 50
            
    if win.current["remote-folder-data"][cur]["changed_active"]:
        for f in win.current["remote-folder-data"][cur]["changed"]:
            
            fdata = win.current["remote-folder-data"][cur]["changed"][f]

            if fdata.get("finished"):
                continue

            icon = "unchecked"
            if fdata["to_download"]:
                icon = "checked"

            def do():
                fdata["to_download"] = not fdata["to_download"]
                
            UI_elements.roundrect(layer, win, 
                          win.current["w"]/2-315,
                          110 + current_Y + win.scroll["http-prompt"],
                          660,
                          40,
                          10,
                          button=do,
                          icon=icon)
            
            
            UI_color.set(layer, win, "text_normal")
            layer.set_font_size(20)
            layer.move_to(win.current["w"]/2-270,
                          current_Y + win.scroll["http-prompt"] + 135)
            layer.show_text(f)

            current_Y += 50
    
    UI_elements.scroll_area(layer, win, "http-prompt", 
        int(win.current["w"]/2-350),
        100,
        700,
        win.current["h"]-260,
        current_Y,
        bar=True,
        mmb=True,
        url="http-server-prompt"
        )
    
    return surface