From 919ed81b90c0644d223cbd1586cab13dcdab01cb Mon Sep 17 00:00:00 2001 From: "Jeison Yehuda Amihud (Blender Dumbass)" Date: Wed, 23 Dec 2020 13:53:55 +0000 Subject: [PATCH] Render Scripts --- network/during_render.py | 304 +++++++++++++++++++++++++++++++++++++ network/network_renders.py | 254 +++++++++++++++++++++++++++++++ 2 files changed, 558 insertions(+) create mode 100644 network/during_render.py create mode 100644 network/network_renders.py diff --git a/network/during_render.py b/network/during_render.py new file mode 100644 index 0000000..865b379 --- /dev/null +++ b/network/during_render.py @@ -0,0 +1,304 @@ +# THIS FILE IS A PART OF VCStudio +# PYTHON 3 + +############################################################################# + +# This file will handle rendering on the background. For more info about how +# it all works. See: +# network/network_renders.py + +############################################################################# + +import os +import sys +import json +import socket +import datetime +from subprocess import * + +# Following 2 functions are copied from the code of Blender-Organizer. How cool +# that python2 and python3 can share so much code. + +def getnumstr(num): + + # This function turns numbers like 1 or 20 into numbers like 0001 or 0020 + + s = "" + for i in range(4-len(str(num))): + s = s + "0" + + return s+str(num) + +def getfileoutput(num, FORMAT): + + # Function gives an output of a file. From the current frame that's rendering. + # instead of having frame 1 and format EXR it will give you 0001.exr + + s = getnumstr(num) + + if FORMAT == "JPEG": + s = s + ".jpg" + else: + s = s + "." + FORMAT.lower() + + return s + +def output(string): + + # This function will act similar to python's print. But rather then just + # printing the string to the terminal. It will also send a signal to the + # VCStudio. And anyone who listening. + + string = str(string) + + print(string) + + for i in range(500): + cs1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + cs1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + cs1.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + cs1.sendto(bytes(string, 'utf-8'), ('127.0.0.1', 54545)) + +# Now to make it all work the script that will lanuch this script will give it +# an agrument of a project location. Which we might need to use for rendering + +project = "" +blender = "" + +if len(sys.argv) > 1: + project = sys.argv[1] + blender = sys.argv[2] + +if not project or not blender: + exit() + +output(project) +output(blender) + +# Now that we have the project. Let's get data about the current renders that +# are setup for rendering. + +def get_active_renders(): + + # This tiny function will gives us the list of current files set to render + # at any moment these will be needed. + + active_renders = [] + try: + active_renders = open(project+"/set/active_renders.data") + active_renders = active_renders.read() + active_renders = active_renders.split("\n") + except: + pass + + return active_renders + +def remove_active_render(render): + + # This function will edit the active renders and remove the current + # render from the list. + + active_renders = open(project+"/set/active_renders.data") + active_renders = active_renders.read() + active_renders = active_renders.split("\n") + + s = open(project+"/set/active_renders.data", "w") + + for i in active_renders: + if i != render and i: + s.write(i+"\n") + + s.close() + +active_renders = get_active_renders() + +# Now I think we also need to get the data from the files. So to speak read their json +# render settings to be able to know what are our start and end frames, format and such. + +def read_settings(filename): + + # This function will read data from the various render settings. + + folder = filename[:filename.rfind("/")]+"/extra" + savefile = folder+filename[filename.rfind("/"):]+".json" + + data = {} + try: + with open(project+savefile) as json_file: + data = json.load(json_file) + except Exception as e: + output(e) + + return data + +# Now let's start the main loop. I'm pretty sure that there will be tons of bugs +# at this stage. So please look at the following code carefully. + +# What I want to do is always read the first line and render it. By the end +# delete the first line from the render list. Remove it from a file. Tho the +# removal could happen at any time anywhere. A user could remove the line +# manually. Or delete the file from the renders in the VCStudio. It doesn't +# matter. This file should be abborted and the next one should start rendering +# to do this. I need to write some clever algorithm. + +while True: + + # I know wild. A while with a True. OMG. But I guess it's the only way to + # insure that it will keep working if there are any current stuff in the + # list. And will stop if there is absolutelly nothing in the list. + + to_break = True # This is our break thing + + active_renders = get_active_renders() # And here we update the current list + + for render in active_renders: + + if render: + + # If anything is found. Then we don't break. And will check again + # on the next go around. This will be deleted from the file by + # the end of rendering this file. + + to_break = False + + output(render) + data = read_settings(render) + output(data) + + # Before we start rendering let's do a couple of things. Mainly + # create the folders and clean them if user so desires. + + folder = render[:render.rfind("/")] + + try: + + os.mkdir(project+folder+"/"+data["save_folder"]) + except: + pass + + try: + if data["clean_folder"]: + + # Okay so if user so desires. We want to wipe the folder clean. + # and so here is the code. + + for filename in os.listdir(project+folder+"/"+data["save_folder"]): + os.remove(project+folder+"/"+data["save_folder"]+"/"+filename) + except: + pass + + # So we have our data. Let's do the rendering. But now so fast. We need + # a brand new while loop here. I know wild. But I need to make sure that + # every frame will be rendered regardless of whether it's crashed ot not. + + # So I will look for all frames currently in the folder. And render the + # first one missing. Always. + + while True: + to_break2 = True + + # We will need to find th + + for frame in range(data["start_frame"], data["end_frame"]+1): + + # I think it's fair to say that some changes to the + + cframe = getfileoutput(frame, data["image_format"] ) + + if cframe not in os.listdir(project+folder+"/"+data["save_folder"]): + + # Basically now we found a missing frame. It could be + # what ever. Where-ever. Usually it's every next frame + # but the user might delete a frame in the middle. And + # will be a missing frame. + + output(cframe) + to_break2 = False + + # Now that we found it I think we car actually render it + + progress = Popen(['stdbuf', '-o0', blender, "-b", + project+render, "-o", + project+folder+"/"+data["save_folder"]+"/####", "-F", + data["image_format"] ,"-f", + str(frame)], stdout=PIPE, universal_newlines=True) + + # Now while the render is not finished. We are going to + # outout everything it saying to the outside. + + # But before we start I want to start counting time. So + # we would have accurate analytics of the renders. + + stf = datetime.datetime.now() + + line = progress.stdout.readline()[:-1] + + while line: + output("VCStudio : RENDERING : "+render+" : "+line) + + # Now at this stage i want it to also listen to a + # command to stop. I might want my CPU back at any + # moment. So let's do this. + + UDP_IP = "127.0.0.1" + UDP_PORT = 54545 + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((UDP_IP, UDP_PORT)) + sock.settimeout(0.05) + input_line, addr = sock.recvfrom(1024) + input_line = input_line.decode('utf8') + sock.close() + + if input_line == "VCStudio : RENDER STOP": + progress.kill() + output("VCStudio : CLOSED") + to_break2 = True + to_break = True + exit() + except Exception as e: + pass + + line = progress.stdout.readline()[:-1] + + # Now that the rendering of the frame is finished. I would + # like to save the analytics data. We are going to use + # microseconds here. + + fif = datetime.datetime.now() + mil = fif - stf + s = int(mil.seconds) + m = int(mil.microseconds) + + thetime = (s*1000000)+m + + # Now I want to record the analytics per folder. So + # data from test renders would not be mangled together + # with data from final renders and so on. + + if data["save_folder"] not in data["analytics"]: + data["analytics"][data["save_folder"]] = {} + + data["analytics"][data["save_folder"]][str(frame)] = thetime + + # And we want to save the file with the analitycs in them + + thefolderis = render[:render.rfind("/")]+"/extra" + savefile = thefolderis+render[render.rfind("/"):]+".json" + + + with open(project+savefile, 'w') as fp: + json.dump(data, fp, indent=4) + + + if to_break2: + break + + remove_active_render(render) + + break + + if to_break: + break diff --git a/network/network_renders.py b/network/network_renders.py new file mode 100644 index 0000000..fb338b7 --- /dev/null +++ b/network/network_renders.py @@ -0,0 +1,254 @@ +# THIS FILE IS A PART OF VCStudio +# PYTHON 3 + +############################################################################## + +# Now you are probably asking yourself. Network? Render? How are these related? +# Actually it's a complicated answer. So let's walk through the idea of the +# Renderer implementation in VCStudio. + +# Why do we need renderer in the first place? Doesn't blender already has one? + +# Well yes. And I would not care if I was always sitting on a biffy, nice +# machine. But back in 2016, 2017 when I was making I'm Not Even Human I realized +# one very ennoying fact. + +# My lap top at the time could not manage the loads of the files. And while +# rendering Blender would crash very often. Of course I was not dumb as was +# rendering into image files and not straight into video. So I could start +# rendering from the last frame. + +# Unfortunatly it was happening way too often. So I started looking at ways to +# unload things from the memory so Blender would crash less. One of those ways +# is rendering using a console. Type blender -b -a and it will launch +# blender without UI saving some memory. Also I could do a simple script that +# restores Blender when if it does crashes. + +# By the time of Blender-Organizer 4.0 I had a system of rendering that will +# look into a folder and see if any file between start and end frame are missing +# and render them. Instead of trying to render every single frame in sequense. + +# Also in Blender-Organizer 4.0 I made a very clever thing. I openned the +# rendering not in a terminal, but rather in it's own process that is piped to +# a little UI window. Where I show some quick analytics. Tho there was one +# problem. Closing the window was closing the blender. Or if not there was no +# way to cancel it. And it would continue till the end or till the crash. + +# So here since VCStudio is a complite re-write of Blender-Organizer I can try +# to re-implement the renderer in a slightly better way. + +# I will run a script that recieves stuff from the pipe and has an ability to +# kill the blender's process. And this script will have no UI. But will instead +# use LOCAL NETWORK to talk to VCStudio. I'm using 127.0.0.1 for this so on +# any normal system even without Internet connection it should work. Since this +# IP adress simply means THIS COMPUTER. + +# I will probably implement some LOCAL NETWORK talk ability later for the +# multiuser. + +# This file is collection of functions for Rendering network sub-system. The UI +# are contained in studio folder. Render it self is a separate file. + +############################################################################## + +import os +import json +import socket + +def read_renders(win): + + # This function will listen for 127.0.0.1 port 54545 for data about current + # renders. NOTE: I'm not doing any encryption and will use UDP protocol. + # so it's quite simple to mess arround with it. NO SECURITY. But I don't see + # any use in hacking it appart from maybe making VCStudio think it's rendering + # a different frame or something. + + # First thing. And you probably think that I'm crazy but bare with me. I + # need to see if a active_renders.data exists. And will read from it on + # each frame. I know. A bit of not cool. But to be honest. The language + # files are also read at every frame. Yeah... + + if not os.path.exists(win.project+"/set/active_renders.data"): + m = open(win.project+"/set/active_renders.data", "w") + m.close() + + # Now we are going to read it on every frame to see that renders are still + # there. + + r = open(win.project+"/set/active_renders.data") + r = r.read() + r = r.split("\n") + + filenames = [] + + for filename in r: + if filename: + + # So basically the file contains list of files currently placed for + # rendering. The renderer will go one by one and when finised will + # remove the filename from this file. Also to cancel the render the + # filename should be removed. The rest will be done using a very + # crappy UDP network protocol on localhost. + + # Now if you just openned the VCStudio. While maybe render were + # doing their job somewhere on the background. You want to re-new + # all of the data. + + # So... + + if filename not in win.renders: + + # We are going to read the JSON file of the render. + + folder = filename[:filename.rfind("/")]+"/extra" + savefile = folder+filename[filename.rfind("/"):]+".json" + + # It might not exits so we are going to do this with try. + + try: + with open(win.project+savefile) as json_file: + data = json.load(json_file) + + win.renders[filename] = data + + except: + pass + + # Now let's make a list of all lines anyway + + if filename not in filenames: + filenames.append(filename) + + # Now let's remove any of them renders that are finished or otherwise + # removed from the file. + + for stuff in list(win.renders.keys()): + + if stuff not in filenames: + del win.renders[stuff] + try: + if stuff == win.current["renders_window"]["filename"]: + win.current["renders_window"]["filename"] = "" + except: + pass + + # Now that we know about the data. Let's read the stream from the network + + string_read = "" + + try: + UDP_IP = "127.0.0.1" + UDP_PORT = 54545 + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((UDP_IP, UDP_PORT)) + + # Now usually it will wait for a message untill one appears. But render + # could be finished. Or some error might accur. Or it could be that no + # render what so ever. So I don't want the UI to freez. So I'm going to + # put a timeout. Very short timeout. + + sock.settimeout(0.005) + + # This comes with it's own problems. Mainly I tested I need to send + # about 500 messages at ones for it to be recognized at all. But it's + # not a big deal what so ever. Just need to keep in mind. 500 messages. + + data, addr = sock.recvfrom(1024) + data = data.decode('utf8') + + sock.close() + + string_read = data + + except: + pass + + # Now let's do something with this data we read from the stream. + # We gonna complitelly ignore anything unrelated. Such as not from + # the project. + + if "network_render_timeout_frame" not in win.current: + win.current["network_render_timeout_frame"] = win.current["frame"] + + if string_read.startswith("VCStudio : "): + if ": RENDERING :" in string_read: + + blend, blend_string = string_read.split(" : ")[2], string_read.split(" : ")[3] + + # So we've got 2 peaces of data from the renderer. Blender. Which + # is a path to the blend file. Similar to the one in + # active_renders.data file that we read previously. + + # And we've got a raw string that blender outputs during render. + # Example: + # Fra:70 Mem:93.09M (Peak 97.78M) | Time:00:01.83 | Rendering 26 / 64 samples + + # You can find a string like this in the top banner inside the blender. + + # From this string we can find out a bunch of things. But not everything is + # that simple. Cycles and Eevee for exmple output different strings. Any + # other, wild render engine will output it's own string. + + # Now I'd like to actually load the data from the JSON file at every step + # like this. Because our rendering script will be recording and saving + # render times of each frame into the file. I need it for analytics. + + if blend in win.renders: + + folder = blend[:blend.rfind("/")]+"/extra" + savefile = folder+blend[blend.rfind("/"):]+".json" + + # It might not exits so we are going to do this with try. + + try: + with open(win.project+savefile) as json_file: + data = json.load(json_file) + + win.renders[blend] = data + except: + pass + + win.renders[blend]["rendering"] = blend_string + + # Next is let's try finding out the current frame rendering. It's + # probably not that hard. Every string usually starts with Fra: + + # So let's try doing it. + frame = 0 + try: + frame = int(blend_string[4:blend_string.find(" ")]) + except: + pass + + win.renders[blend]["current_frame"] = frame + + # The rest of it I will do in the UI. + + win.current["network_render_timeout_frame"] = win.current["frame"] + + + # Now sometimes for no particular reason the whole rendering thing could crash + # or in other words. If there is no data sent. We want to clean the renderings + # and make it so the user sees that renders DO NOT render. For this there should + # be a timeout system. Basically if in a given amount of frames there is no message + # from the renderer script. We going to say that it's not rendering. + + if win.current["frame"] - win.current["network_render_timeout_frame"] > 100: + for blend in win.renders: + win.renders[blend]["rendering"] = False + +def stop_render(win): + + # This function will stop the rendering. It will bombard the renderer with + # stop messages. + + for i in range(5000): + cs1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + cs1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + cs1.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + cs1.sendto(bytes("VCStudio : RENDER STOP", 'utf-8'), ('127.0.0.1', 54545)) + + +