Madiator2011/Blender-Pipeline
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Render Scripts

Browse source
This commit is contained in:
Jeison Yehuda Amihud (Blender Dumbass) 2020-12-23 13:53:55 +00:00
parent 7f22fb6bb6
commit 919ed81b90
2 changed files with 558 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

304
network/during_render.py Normal file
View file

@ -0,0 +1,304 @@
# THIS FILE IS A PART OF VCStudio
# PYTHON 3
#############################################################################
# This file will handle rendering on the background. For more info about how
# it all works. See:
# network/network_renders.py
#############################################################################
import os
import sys
import json
import socket
import datetime
from subprocess import *
# Following 2 functions are copied from the code of Blender-Organizer. How cool
# that python2 and python3 can share so much code.
def getnumstr(num):
# This function turns numbers like 1 or 20 into numbers like 0001 or 0020
s = ""
for i in range(4-len(str(num))):
s = s + "0"
return s+str(num)
def getfileoutput(num, FORMAT):
# Function gives an output of a file. From the current frame that's rendering.
# instead of having frame 1 and format EXR it will give you 0001.exr
s = getnumstr(num)
if FORMAT == "JPEG":
s = s + ".jpg"
else:
s = s + "." + FORMAT.lower()
return s
def output(string):
# This function will act similar to python's print. But rather then just
# printing the string to the terminal. It will also send a signal to the
# VCStudio. And anyone who listening.
string = str(string)
print(string)
for i in range(500):
cs1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
cs1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
cs1.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
cs1.sendto(bytes(string, 'utf-8'), ('127.0.0.1', 54545))
# Now to make it all work the script that will lanuch this script will give it
# an agrument of a project location. Which we might need to use for rendering
project = ""
blender = ""
if len(sys.argv) > 1:
project = sys.argv[1]
blender = sys.argv[2]
if not project or not blender:
exit()
output(project)
output(blender)
# Now that we have the project. Let's get data about the current renders that
# are setup for rendering.
def get_active_renders():
# This tiny function will gives us the list of current files set to render
# at any moment these will be needed.
active_renders = []
try:
active_renders = open(project+"/set/active_renders.data")
active_renders = active_renders.read()
active_renders = active_renders.split("\n")
except:
pass
return active_renders
def remove_active_render(render):
# This function will edit the active renders and remove the current
# render from the list.
active_renders = open(project+"/set/active_renders.data")
active_renders = active_renders.read()
active_renders = active_renders.split("\n")
s = open(project+"/set/active_renders.data", "w")
for i in active_renders:
if i != render and i:
s.write(i+"\n")
s.close()
active_renders = get_active_renders()
# Now I think we also need to get the data from the files. So to speak read their json
# render settings to be able to know what are our start and end frames, format and such.
def read_settings(filename):
# This function will read data from the various render settings.
folder = filename[:filename.rfind("/")]+"/extra"
savefile = folder+filename[filename.rfind("/"):]+".json"
data = {}
try:
with open(project+savefile) as json_file:
data = json.load(json_file)
except Exception as e:
output(e)
return data
# Now let's start the main loop. I'm pretty sure that there will be tons of bugs
# at this stage. So please look at the following code carefully.
# What I want to do is always read the first line and render it. By the end
# delete the first line from the render list. Remove it from a file. Tho the
# removal could happen at any time anywhere. A user could remove the line
# manually. Or delete the file from the renders in the VCStudio. It doesn't
# matter. This file should be abborted and the next one should start rendering
# to do this. I need to write some clever algorithm.
while True:
# I know wild. A while with a True. OMG. But I guess it's the only way to
# insure that it will keep working if there are any current stuff in the
# list. And will stop if there is absolutelly nothing in the list.
to_break = True # This is our break thing
active_renders = get_active_renders() # And here we update the current list
for render in active_renders:
if render:
# If anything is found. Then we don't break. And will check again
# on the next go around. This will be deleted from the file by
# the end of rendering this file.
to_break = False
output(render)
data = read_settings(render)
output(data)
# Before we start rendering let's do a couple of things. Mainly
# create the folders and clean them if user so desires.
folder = render[:render.rfind("/")]
try:
os.mkdir(project+folder+"/"+data["save_folder"])
except:
pass
try:
if data["clean_folder"]:
# Okay so if user so desires. We want to wipe the folder clean.
# and so here is the code.
for filename in os.listdir(project+folder+"/"+data["save_folder"]):
os.remove(project+folder+"/"+data["save_folder"]+"/"+filename)
except:
pass
# So we have our data. Let's do the rendering. But now so fast. We need
# a brand new while loop here. I know wild. But I need to make sure that
# every frame will be rendered regardless of whether it's crashed ot not.
# So I will look for all frames currently in the folder. And render the
# first one missing. Always.
while True:
to_break2 = True
# We will need to find th
for frame in range(data["start_frame"], data["end_frame"]+1):
# I think it's fair to say that some changes to the
cframe = getfileoutput(frame, data["image_format"] )
if cframe not in os.listdir(project+folder+"/"+data["save_folder"]):
# Basically now we found a missing frame. It could be
# what ever. Where-ever. Usually it's every next frame
# but the user might delete a frame in the middle. And
# will be a missing frame.
output(cframe)
to_break2 = False
# Now that we found it I think we car actually render it
progress = Popen(['stdbuf', '-o0', blender, "-b",
project+render, "-o",
project+folder+"/"+data["save_folder"]+"/####", "-F",
data["image_format"] ,"-f",
str(frame)], stdout=PIPE, universal_newlines=True)
# Now while the render is not finished. We are going to
# outout everything it saying to the outside.
# But before we start I want to start counting time. So
# we would have accurate analytics of the renders.
stf = datetime.datetime.now()
line = progress.stdout.readline()[:-1]
while line:
output("VCStudio : RENDERING : "+render+" : "+line)
# Now at this stage i want it to also listen to a
# command to stop. I might want my CPU back at any
# moment. So let's do this.
UDP_IP = "127.0.0.1"
UDP_PORT = 54545
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((UDP_IP, UDP_PORT))
sock.settimeout(0.05)
input_line, addr = sock.recvfrom(1024)
input_line = input_line.decode('utf8')
sock.close()
if input_line == "VCStudio : RENDER STOP":
progress.kill()
output("VCStudio : CLOSED")
to_break2 = True
to_break = True
exit()
except Exception as e:
pass
line = progress.stdout.readline()[:-1]
# Now that the rendering of the frame is finished. I would
# like to save the analytics data. We are going to use
# microseconds here.
fif = datetime.datetime.now()
mil = fif - stf
s = int(mil.seconds)
m = int(mil.microseconds)
thetime = (s*1000000)+m
# Now I want to record the analytics per folder. So
# data from test renders would not be mangled together
# with data from final renders and so on.
if data["save_folder"] not in data["analytics"]:
data["analytics"][data["save_folder"]] = {}
data["analytics"][data["save_folder"]][str(frame)] = thetime
# And we want to save the file with the analitycs in them
thefolderis = render[:render.rfind("/")]+"/extra"
savefile = thefolderis+render[render.rfind("/"):]+".json"
with open(project+savefile, 'w') as fp:
json.dump(data, fp, indent=4)
if to_break2:
break
remove_active_render(render)
break
if to_break:
break

254
network/network_renders.py Normal file
View file

@ -0,0 +1,254 @@
# THIS FILE IS A PART OF VCStudio
# PYTHON 3
##############################################################################
# Now you are probably asking yourself. Network? Render? How are these related?
# Actually it's a complicated answer. So let's walk through the idea of the
# Renderer implementation in VCStudio.
# Why do we need renderer in the first place? Doesn't blender already has one?
# Well yes. And I would not care if I was always sitting on a biffy, nice
# machine. But back in 2016, 2017 when I was making I'm Not Even Human I realized
# one very ennoying fact.
# My lap top at the time could not manage the loads of the files. And while
# rendering Blender would crash very often. Of course I was not dumb as was
# rendering into image files and not straight into video. So I could start
# rendering from the last frame.
# Unfortunatly it was happening way too often. So I started looking at ways to
# unload things from the memory so Blender would crash less. One of those ways
# is rendering using a console. Type blender -b <filename> -a and it will launch
# blender without UI saving some memory. Also I could do a simple script that
# restores Blender when if it does crashes.
# By the time of Blender-Organizer 4.0 I had a system of rendering that will
# look into a folder and see if any file between start and end frame are missing
# and render them. Instead of trying to render every single frame in sequense.
# Also in Blender-Organizer 4.0 I made a very clever thing. I openned the
# rendering not in a terminal, but rather in it's own process that is piped to
# a little UI window. Where I show some quick analytics. Tho there was one
# problem. Closing the window was closing the blender. Or if not there was no
# way to cancel it. And it would continue till the end or till the crash.
# So here since VCStudio is a complite re-write of Blender-Organizer I can try
# to re-implement the renderer in a slightly better way.
# I will run a script that recieves stuff from the pipe and has an ability to
# kill the blender's process. And this script will have no UI. But will instead
# use LOCAL NETWORK to talk to VCStudio. I'm using 127.0.0.1 for this so on
# any normal system even without Internet connection it should work. Since this
# IP adress simply means THIS COMPUTER.
# I will probably implement some LOCAL NETWORK talk ability later for the
# multiuser.
# This file is collection of functions for Rendering network sub-system. The UI
# are contained in studio folder. Render it self is a separate file.
##############################################################################
import os
import json
import socket
def read_renders(win):
# This function will listen for 127.0.0.1 port 54545 for data about current
# renders. NOTE: I'm not doing any encryption and will use UDP protocol.
# so it's quite simple to mess arround with it. NO SECURITY. But I don't see
# any use in hacking it appart from maybe making VCStudio think it's rendering
# a different frame or something.
# First thing. And you probably think that I'm crazy but bare with me. I
# need to see if a active_renders.data exists. And will read from it on
# each frame. I know. A bit of not cool. But to be honest. The language
# files are also read at every frame. Yeah...
if not os.path.exists(win.project+"/set/active_renders.data"):
m = open(win.project+"/set/active_renders.data", "w")
m.close()
# Now we are going to read it on every frame to see that renders are still
# there.
r = open(win.project+"/set/active_renders.data")
r = r.read()
r = r.split("\n")
filenames = []
for filename in r:
if filename:
# So basically the file contains list of files currently placed for
# rendering. The renderer will go one by one and when finised will
# remove the filename from this file. Also to cancel the render the
# filename should be removed. The rest will be done using a very
# crappy UDP network protocol on localhost.
# Now if you just openned the VCStudio. While maybe render were
# doing their job somewhere on the background. You want to re-new
# all of the data.
# So...
if filename not in win.renders:
# We are going to read the JSON file of the render.
folder = filename[:filename.rfind("/")]+"/extra"
savefile = folder+filename[filename.rfind("/"):]+".json"
# It might not exits so we are going to do this with try.
try:
with open(win.project+savefile) as json_file:
data = json.load(json_file)
win.renders[filename] = data
except:
pass
# Now let's make a list of all lines anyway
if filename not in filenames:
filenames.append(filename)
# Now let's remove any of them renders that are finished or otherwise
# removed from the file.
for stuff in list(win.renders.keys()):
if stuff not in filenames:
del win.renders[stuff]
try:
if stuff == win.current["renders_window"]["filename"]:
win.current["renders_window"]["filename"] = ""
except:
pass
# Now that we know about the data. Let's read the stream from the network
string_read = ""
try:
UDP_IP = "127.0.0.1"
UDP_PORT = 54545
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((UDP_IP, UDP_PORT))
# Now usually it will wait for a message untill one appears. But render
# could be finished. Or some error might accur. Or it could be that no
# render what so ever. So I don't want the UI to freez. So I'm going to
# put a timeout. Very short timeout.
sock.settimeout(0.005)
# This comes with it's own problems. Mainly I tested I need to send
# about 500 messages at ones for it to be recognized at all. But it's
# not a big deal what so ever. Just need to keep in mind. 500 messages.
data, addr = sock.recvfrom(1024)
data = data.decode('utf8')
sock.close()
string_read = data
except:
pass
# Now let's do something with this data we read from the stream.
# We gonna complitelly ignore anything unrelated. Such as not from
# the project.
if "network_render_timeout_frame" not in win.current:
win.current["network_render_timeout_frame"] = win.current["frame"]
if string_read.startswith("VCStudio : "):
if ": RENDERING :" in string_read:
blend, blend_string = string_read.split(" : ")[2], string_read.split(" : ")[3]
# So we've got 2 peaces of data from the renderer. Blender. Which
# is a path to the blend file. Similar to the one in
# active_renders.data file that we read previously.
# And we've got a raw string that blender outputs during render.
# Example:
# Fra:70 Mem:93.09M (Peak 97.78M) | Time:00:01.83 | Rendering 26 / 64 samples
# You can find a string like this in the top banner inside the blender.
# From this string we can find out a bunch of things. But not everything is
# that simple. Cycles and Eevee for exmple output different strings. Any
# other, wild render engine will output it's own string.
# Now I'd like to actually load the data from the JSON file at every step
# like this. Because our rendering script will be recording and saving
# render times of each frame into the file. I need it for analytics.
if blend in win.renders:
folder = blend[:blend.rfind("/")]+"/extra"
savefile = folder+blend[blend.rfind("/"):]+".json"
# It might not exits so we are going to do this with try.
try:
with open(win.project+savefile) as json_file:
data = json.load(json_file)
win.renders[blend] = data
except:
pass
win.renders[blend]["rendering"] = blend_string
# Next is let's try finding out the current frame rendering. It's
# probably not that hard. Every string usually starts with Fra:<number>
# So let's try doing it.
frame = 0
try:
frame = int(blend_string[4:blend_string.find(" ")])
except:
pass
win.renders[blend]["current_frame"] = frame
# The rest of it I will do in the UI.
win.current["network_render_timeout_frame"] = win.current["frame"]
# Now sometimes for no particular reason the whole rendering thing could crash
# or in other words. If there is no data sent. We want to clean the renderings
# and make it so the user sees that renders DO NOT render. For this there should
# be a timeout system. Basically if in a given amount of frames there is no message
# from the renderer script. We going to say that it's not rendering.
if win.current["frame"] - win.current["network_render_timeout_frame"] > 100:
for blend in win.renders:
win.renders[blend]["rendering"] = False
def stop_render(win):
# This function will stop the rendering. It will bombard the renderer with
# stop messages.
for i in range(5000):
cs1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
cs1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
cs1.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
cs1.sendto(bytes("VCStudio : RENDER STOP", 'utf-8'), ('127.0.0.1', 54545))