Blender-Pipeline/network/http_client.py
2023-12-22 00:10:57 +02:00

876 lines
26 KiB
Python

# (c) J.Y.Amihud 2023
# GPL-3 or any later version
import os
import json
import zlib
import time
import fnmatch
import threading
import hashlib
import urllib.request
import urllib.parse
import subprocess
# GTK module ( Graphical interface
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from gi.repository import GLib
from gi.repository import Gdk
import cairo
# Own modules
from settings import settings
from settings import fileformats
from settings import talk
from project_manager import pm_project
#UI modules
from UI import UI_elements
from UI import UI_color
from UI import UI_math
from studio import story
from studio import analytics
# This is a http client. To work on projects remotely.
def csize(x):
x = float(x)
l = ["B","KB", "MB", "GB", "TB"]
for i in range(5):
if x > 1024:
x = x / 1024
else:
return str(round(x, 2))+" "+l[i]
return str(round(x, 2))+" "+l[i]
def go(win, website, path):
url = website+path
print("url", url)
response = urllib.request.urlopen(url)
data = response.read()
text = data.decode("utf-8")
data = json.loads(text)
return data
def down(win, website, filename, fsize=100000):
# If it is a blend file, we are in trouble.
# if filename.endswith(".blend"):
# dp = go(win, website, "/blend"+filename)
# dplen = len(dp.get("files", {}))
# for f in dp.get("files", {}):
# try:
# hashis = hashlib.md5(open(win.project+f,'rb').read()).hexdigest()
# except:
# hashis = ""
# if hashis != dp.get("files", {}).get(f, {}).get("md5"):
# win.current["http-server"]["message"] = f[f.rfind("/")+1:]
# filesize = dp.get("files", {}).get(f, {}).get("filesize")
# print("trying to get:", f)
# pp = down(win, website, f, filesize)
# Making folder for the files
try:
os.makedirs(win.project+filename[:filename.rfind("/")+1])
except:
pass
# downloading the file
url = website+"/download_z"+filename
response = urllib.request.urlopen(url)
for line in str(response.info()).split("\n"):
if line.startswith("Content-length: "):
try:
fsize = int(line[line.find(" ")+1:])
except:
pass
savef = open(win.project+filename, "wb")
zd = b""
sofar = 0
chunk = response.read(8192)
while chunk:
zd = zd + chunk
chunk = response.read(8192)
sofar = sofar + 8192
win.current["http-server"]["message"] = filename[filename.rfind("/")+1:]
win.current["http-server"]["fileprog"] = sofar / fsize
unz = zlib.decompress(zd)
savef.write(unz)
savef.close()
def download_missing_changed(win, cur, call):
# This function downloads files from remote server using the
# remote-folder-data data.
website = win.analytics["remote-server-url"]
for t in ["missing", "changed"]:
while win.current["remote-folder-data"][cur][t]: # IDK WHAT I'M DOING!!!
try:
for f in win.current["remote-folder-data"][cur][t]:
fdata = win.current["remote-folder-data"][cur][t][f]
if fdata.get("to_download", True):
print("Downloading", f)
try:
down(win, website, f, fdata.get("filesize", 10000))
except Exception as e:
print(e)
del win.current["remote-folder-data"][cur][t][f]
except:
pass
win.current["remote-folder-data"][cur]["downloading"] = False
win.current["calls"][call]["var"] = False
def get_folder_info(win, folders, cur):
# This function gets an information of what files should be changed. And outputs
# the found data into the the global win thing, so it could accessed later.
# Making sure we have the infrastructure.
print("Getting Update Info About Folders")
website = win.analytics["remote-server-url"]
if "remote-folder-data" not in win.current:
win.current["remote-folder-data"] = {}
if cur not in win.current["remote-folder-data"]:
win.current["remote-folder-data"][cur] = {"missing":{},
"changed":{}}
win.current["remote-folder-data"][cur]["loaded"] = False
def sort_file(filename, files, f, filedata=None):
if not filedata:
filedata = files.get(f, {})
filedata["finished"] = False
try:
hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
except:
hashis = ""
if not os.path.exists(win.project+filename):
win.current["remote-folder-data"][cur]["missing"][filename] = filedata
elif hashis != filedata.get("md5"):
filedata["to_download"] = True
win.current["remote-folder-data"][cur]["changed"][filename] = filedata
# Part one: Getting all the data about the files.
for foldername in folders:
try:
answer = go(win, website, "/list"+foldername)
files = list(answer.get("files", {}).keys())
for f in fnmatch.filter(files, folders.get(foldername, "*")):
# Now we need to figure out whether we have the file
# , whether we don't even have it ,
# or whether we have a different version.
fullfilename = foldername+"/"+f
sort_file(fullfilename, answer.get("files", {}), f)
except Exception as e:
error = "Error: "+ str(e)
print(error)
# Part two: Unwrapping all the blend dependancies data.
# Here we are going to store blend file that were checked already.
win.current["remote-folder-data"][cur]["blend_checked"] = []
# A function checking if we still have some unchecked .blends
def not_checked():
for t in ["missing", "changed"]:
for i in win.current["remote-folder-data"][cur][t]:
if i not in win.current["remote-folder-data"][cur]["blend_checked"] and i.endswith(".blend"):
return i
return False
nc = not_checked()
while nc:
try:
dp = go(win, website, "/blend"+nc)
for f in dp.get("files", {}):
try:
sort_file(f,
dp.get("files", {}),
f[f.rfind("/")+1:],
filedata=dp.get("files", {})[f])
except Exception as e:
print(e)
except Exception as e:
print(e)
win.current["remote-folder-data"][cur]["blend_checked"].append(nc)
nc = not_checked()
for t in ["missing", "changed"]:
print(t+":")
for i in win.current["remote-folder-data"][cur][t]:
print(" "+i)
win.current["remote-folder-data"][cur]["loaded"] = True
def get_folders(win):
# This function will get specified subfolders
win.current["http-server"]["message"] = "Getting Remote Folders"
folder = win.current["http-server"]["args"]
website = win.analytics["remote-server-url"]
call = win.current["http-server"]["call"]
try:
answer = go(win, website, "/list"+folder)
for n, i in enumerate(answer.get("folders", [])):
win.current["http-server"]["progress"] = n/len(answer.get("folders", []))
win.current["http-server"]["message"] = i
try:
os.makedirs(win.project+folder+"/"+i)
except:
pass
except Exception as e:
error = "Error: "+ str(e)
print(error)
win.current["http-server"]["message"] = error
time.sleep(1)
# Ending the process
win.current["calls"][call]["var"] = False
def get_asset_files(win):
# This function will get specified subfolders
win.current["http-server"]["message"] = "Getting / Updating Files"
folder = win.current["http-server"]["args"]
website = win.analytics["remote-server-url"]
call = win.current["http-server"]["call"]
folder = folder.replace("//", "/")
if folder.endswith("/"):
folder = folder[:-1]
try:
answer = go(win, website, "/list"+folder)
for n, i in enumerate(answer.get("files", [])):
win.current["http-server"]["progress"] = n/len(answer.get("files", []))
win.current["http-server"]["message"] = i
filename = folder+"/"+i
try:
hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
except:
hashis = ""
if hashis != answer.get("files", {}).get(i, {}).get("md5"):
filesize = answer.get("files", {}).get(i, {}).get("filesize")
down(win, website, filename, filesize)
win.current["http-server"]["message"] = "Getting Thumbnail"
other_folder = folder+"/renders"
print("otherfolder 1", other_folder)
answer = go(win, website, "/list"+other_folder)
for n, i in enumerate(answer.get("files", [])):
win.current["http-server"]["progress"] = n/len(answer.get("files", []))
win.current["http-server"]["message"] = i
if i.startswith("Preview"):
print("getting", i)
filename = other_folder+"/"+i
try:
hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
except:
hashis = ""
if hashis != answer.get("files", {}).get(i, {}).get("md5"):
filesize = answer.get("files", {}).get(i, {}).get("filesize")
down(win, website, filename, filesize)
win.current["http-server"]["message"] = "Getting Asset Blend"
other_folder = folder[:folder.rfind("/")+1].replace("/dev/", "/ast/")
print("otherfolder 2", other_folder, folder)
answer = go(win, website, "/list"+other_folder)
for n, i in enumerate(answer.get("files", [])):
win.current["http-server"]["progress"] = n/len(answer.get("files", []))
win.current["http-server"]["message"] = i
if i.startswith(folder[folder.rfind("/")+1:]):
print("getting", i)
filename = other_folder+"/"+i
try:
hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
except:
hashis = ""
if hashis != answer.get("files", {}).get(i, {}).get("md5"):
filesize = answer.get("files", {}).get(i, {}).get("filesize")
down(win, website, filename, filesize)
except Exception as e:
error = "Error: "+ str(e)
print(error)
win.current["http-server"]["message"] = error
time.sleep(1)
# Ending the process
win.current["calls"][call]["var"] = False
def get_files(win):
# This function will get specified subfolders
win.current["http-server"]["message"] = "Getting / Updating Files"
folder = win.current["http-server"]["args"]
website = win.analytics["remote-server-url"]
call = win.current["http-server"]["call"]
try:
answer = go(win, website, "/list"+folder)
for n, i in enumerate(answer.get("files", [])):
win.current["http-server"]["progress"] = n/len(answer.get("files", []))
win.current["http-server"]["message"] = i
filename = folder+"/"+i
try:
hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
except:
print("missing", filename)
hashis = ""
if hashis != answer.get("files", {}).get(i, {}).get("md5"):
filesize = answer.get("files", {}).get(i, {}).get("filesize")
down(win, website, filename, filesize)
except Exception as e:
error = "Error: "+ str(e)
print(error)
win.current["http-server"]["message"] = error
time.sleep(1)
# Ending the process
win.current["calls"][call]["var"] = False
def get_essentials(win):
# This function will get essential parts of the project.
# Essentials are: Story, Analitycs, Project.progress and Banner Image.
essentials = [
"/pln/story.vcss",
"/set/analytics.json",
"/set/project.progress",
"/set/banner.png"]
website = win.analytics["remote-server-url"]
call = win.current["http-server"]["call"]
for n, i in enumerate(essentials):
win.current["http-server"]["progress"] = n/len(essentials)
win.current["http-server"]["message"] = i
try:
down(win, website, i)
except Exception as e:
error = "Error: "+ str(e)
print(error)
win.current["http-server"]["message"] = error
time.sleep(1)
win.story = story.load(win.project)
win.analytics = analytics.load(win.project)
win.analytics["remote-server-url"] = website
win.analytics["from-remote-server"] = True
analytics.save(win.project, win.analytics)
# Ending the process
win.current["calls"][call]["var"] = False
def layer(win, call):
# Making the layer
surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, win.current['w'],
win.current['h'])
layer = cairo.Context(surface)
#text setting
layer.select_font_face("Monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
UI_color.set(layer, win, "dark_overdrop")
layer.rectangle(
0,
0,
win.current["w"],
win.current["h"],
)
layer.fill()
# So it's going to be like a little window in the center of the VCStudio
# with a simple UI. Probably like 2 things. Folder and a projectname.
UI_color.set(layer, win, "node_background")
UI_elements.roundrect(layer, win,
win.current["w"]/2-250,
win.current["h"]/2-60,
500,
150,
10)
# Title of the operation. Incase the user forgot.
UI_elements.text(layer, win, "scan_project_title",
win.current["w"]/2-250,
win.current["h"]/2-35,
500,
30,
10,
fill=False,
centered=True,
editable=False)
win.text["scan_project_title"]["text"] = win.current["http-server"]["message"]
# File Progressbar
UI_color.set(layer, win, "progress_background")
UI_elements.roundrect(layer, win,
win.current["w"]/2-200,
win.current["h"]/2+10,
400,
20,
10,
tip="Hello")
UI_color.set(layer, win, "progress_active")
UI_elements.roundrect(layer, win,
win.current["w"]/2-200,
win.current["h"]/2+10,
400*max(0, min(1, win.current["http-server"]["fileprog"])),
20,
10,)
# Progressbar
UI_color.set(layer, win, "progress_background")
UI_elements.roundrect(layer, win,
win.current["w"]/2-200,
win.current["h"]/2+40,
400,
20,
10,
tip="Hello")
UI_color.set(layer, win, "progress_active")
UI_elements.roundrect(layer, win,
win.current["w"]/2-200,
win.current["h"]/2+40,
400*max(0, min(1, win.current["http-server"]["progress"])),
20,
10,)
return surface
def prompt_layer(win, call):
# Making the layer
surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, win.current['w'],
win.current['h'])
layer = cairo.Context(surface)
#text setting
layer.select_font_face("Monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
UI_color.set(layer, win, "dark_overdrop")
layer.rectangle(
0,
0,
win.current["w"],
win.current["h"],
)
layer.fill()
# So it's going to be like a little window in the center of the VCStudio
# with a simple UI. Probably like 2 things. Folder and a projectname.
UI_color.set(layer, win, "node_background")
UI_elements.roundrect(layer, win,
win.current["w"]/2-350,
100,
700,
win.current["h"]-200,
10)
cur = win.current["remote-folder-data"]["prompt"]
if "downloading" not in win.current["remote-folder-data"][cur]:
win.current["remote-folder-data"][cur]["downloading"] = False
if not win.current["remote-folder-data"][cur]["downloading"]:
# Exit button
def do():
win.current["calls"][call]["var"] = False
UI_elements.roundrect(layer, win,
win.current["w"]/2+310,
win.current["h"]-140,
40,
40,
10,
button=do,
icon="cancel",
tip=talk.text("cancel"))
# Download button
if win.current["remote-folder-data"][cur]["loaded"]:
def do():
win.current["remote-folder-data"][cur]["downloading"] = True
start_downaloding = threading.Thread(target=download_missing_changed,
args=(win, cur, call, ))
start_downaloding.setDaemon(True)
start_downaloding.start()
UI_elements.roundrect(layer, win,
win.current["w"]/2+270,
win.current["h"]-140,
40,
40,
10,
button=do,
icon="download",
tip=talk.text("DownloadRemoteServer"))
else:
UI_color.set(layer, win, "text_normal")
layer.set_font_size(15)
layer.move_to(win.current["w"]/2,
win.current["h"]-120,)
layer.show_text(talk.text("StillLoadingRemoteData"))
else:
# Progress bar
UI_color.set(layer, win, "progress_background")
UI_elements.roundrect(layer, win,
win.current["w"]/2-200,
win.current["h"]-140,
400,
20,
10,
tip="Hello")
UI_color.set(layer, win, "progress_active")
UI_elements.roundrect(layer, win,
win.current["w"]/2-200,
win.current["h"]-140,
400*max(0, min(1, win.current["http-server"]["fileprog"])),
20,
10,)
UI_color.set(layer, win, "text_normal")
layer.set_font_size(15)
layer.move_to(win.current["w"]/2-180,
win.current["h"]-145)
layer.show_text(win.current["http-server"]["message"])
# Top Tabs
if "active_tab" not in win.current["remote-folder-data"][cur]:
win.current["remote-folder-data"][cur]["active_tab"] = "changed"
# Missing
def do():
win.current["remote-folder-data"][cur]["active_tab"] = "missing"
if win.current["remote-folder-data"][cur]["active_tab"] == "missing":
UI_color.set(layer, win, "progress_time")
UI_elements.roundrect(layer, win,
win.current["w"]/2-340,
110,
330,
40,
10)
UI_elements.roundrect(layer, win,
win.current["w"]/2-340,
110,
330,
40,
10,
button=do,
icon="new_file",
tip=talk.text("MissingFiles"))
UI_color.set(layer, win, "text_normal")
layer.set_font_size(20)
layer.move_to(win.current["w"]/2-290,
135)
layer.show_text(str(len(win.current["remote-folder-data"][cur]["missing"]))+" "+talk.text("MissingFiles"))
# Changed
def do():
win.current["remote-folder-data"][cur]["active_tab"] = "changed"
if win.current["remote-folder-data"][cur]["active_tab"] == "changed":
UI_color.set(layer, win, "progress_time")
UI_elements.roundrect(layer, win,
win.current["w"]/2+10,
110,
330,
40,
10)
UI_elements.roundrect(layer, win,
win.current["w"]/2+10,
110,
330,
40,
10,
button=do,
icon="configure_file",
tip=talk.text("ChangedFiles"))
UI_color.set(layer, win, "text_normal")
layer.set_font_size(20)
layer.move_to(win.current["w"]/2+60,
135)
layer.show_text(str(len(win.current["remote-folder-data"][cur]["changed"]))+" "+talk.text("ChangedFiles"))
# Search
UI_elements.image(layer, win, "settings/themes/"\
+win.settings["Theme"]+"/icons/search.png",
win.current["w"]/2,
160,
40,
40)
UI_elements.text(layer, win, "http_client_search",
win.current["w"]/2+50,
160,
250,
40)
# Toggle Visible
if "toggle_visible" not in win.current["remote-folder-data"][cur]:
win.current["remote-folder-data"][cur]["toggle_visible"] = True
toggle = {"do":None}
def do():
win.current["remote-folder-data"][cur]["toggle_visible"] = not win.current["remote-folder-data"][cur]["toggle_visible"]
toggle["do"] = win.current["remote-folder-data"][cur]["toggle_visible"]
icon = "unchecked"
if win.current["remote-folder-data"][cur]["toggle_visible"]:
icon = "checked"
UI_elements.roundrect(layer, win,
win.current["w"]/2-340,
160,
330,
40,
10,
button=do,
icon=icon,
tip=talk.text("ToggleVisible"))
UI_color.set(layer, win, "text_normal")
layer.set_font_size(20)
layer.move_to(win.current["w"]/2-290,
185)
layer.show_text(talk.text("ToggleVisible"))
# Clipping everything
UI_elements.roundrect(layer, win,
win.current["w"]/2-350,
200,
700,
win.current["h"]-360,
10,
fill=False)
layer.clip()
clip = [
win.current["w"]/2-350,
200,
700,
win.current["h"]-360]
# Setting up the scroll
if "http-prompt" not in win.scroll:
win.scroll["http-prompt"] = 0
current_Y = 100
# Display list
DisplayList = {}
ActiveTab = win.current["remote-folder-data"][cur]["active_tab"]
DisplayList = win.current["remote-folder-data"][cur][ActiveTab]
try:
for f in DisplayList:
fdata = DisplayList[f]
if fdata.get("finished"):
continue
if win.text["http_client_search"]["text"] and not UI_math.found(f, win.text["http_client_search"]["text"]):
continue
ticon = "unchecked"
if fdata.get("to_download", True):
ticon = "checked"
if toggle["do"] != None:
fdata["to_download"] = toggle["do"]
def do():
fdata["to_download"] = not fdata.get("to_download", True)
# Filetype icon
ext = f[f.rfind(".")+1:]
if ext in fileformats.images:
icon = "image"
elif ext in fileformats.videos:
icon = "video"
elif ext in fileformats.sounds:
icon = "mus"
elif ext == "blend":
icon = "blender"
elif ext == "progress":
icon = "checklist"
else:
icon = "file"
UI_elements.roundrect(layer, win,
win.current["w"]/2-315,
110 + current_Y + win.scroll["http-prompt"],
660,
40,
10,
button=do,
icon=ticon,
tip=f)
UI_elements.image(layer, win, "settings/themes/"+win.settings["Theme"]+"/icons/"+icon+".png",
win.current["w"]/2-275,
110 + current_Y + win.scroll["http-prompt"],
40,
40)
UI_color.set(layer, win, "text_normal")
layer.set_font_size(20)
layer.move_to(win.current["w"]/2-220,
current_Y + win.scroll["http-prompt"] + 135)
layer.show_text(f[f.rfind("/")+1:])
current_Y += 50
except Exception as e:
print(e)
pass
UI_elements.scroll_area(layer, win, "http-prompt",
int(win.current["w"]/2-350),
200,
700,
win.current["h"]-360,
current_Y,
bar=True,
mmb=True,
url="http-server-prompt"
)
return surface