Add files via upload
archfan
GitHub
parent
f19ef4f198
commit
497ecce052
|
|
@ -0,0 +1,9 @@
|
|||
# Blender Lipsync Addon
|
||||
|
||||
This addon does not do any lipsyncing by itself, but applies existing phoneme data generated with Allosaurus, a wonderful Python module. It is presently targeted at the MecaFace facial animation rig but may be useful for other rigs, and may be expanded to be general-purpose in the future.
|
||||
|
||||
For our purposes I've written a Discord bot for this, but pasting the output of:
|
||||
|
||||
`python -m allosaurus.run -i <your WAV file> --model eng2102 --lang eng --timestamp=True`
|
||||
|
||||
into a new file with the extension `.sync` should also give you what you need.
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
bl_info = {
|
||||
"name": "Automated Lipsync",
|
||||
"description" : "Automates lipsyncing for MecaFace facial animation rigs.",
|
||||
"version" : (0, 0, 5),
|
||||
"blender": (2, 80, 0),
|
||||
"category": "Animation",
|
||||
}
|
||||
|
||||
import bpy
|
||||
from bpy.types import (Panel, Operator)
|
||||
from bpy_extras.io_utils import ImportHelper
|
||||
from bpy.props import (StringProperty, IntProperty, EnumProperty, BoolProperty)
|
||||
from bpy.utils import (register_class, unregister_class)
|
||||
import json
|
||||
|
||||
phones = {
|
||||
"a" : "A",
|
||||
"aː" : "A",
|
||||
"b" : "L",
|
||||
"d" : "L",
|
||||
"d̠" : "L",
|
||||
"e" : "E",
|
||||
"eː" : "E",
|
||||
"e̞" : "E",
|
||||
"f" : "F",
|
||||
"h" : "A",
|
||||
"i" : "A",
|
||||
"iː" : "A",
|
||||
"j" : "L",
|
||||
"k" : "S",
|
||||
"kʰ" : "S",
|
||||
"l" : "L",
|
||||
"m" : "M",
|
||||
"n" : "S",
|
||||
"o" : "O",
|
||||
"oː" : "O",
|
||||
"p" : "F",
|
||||
"pʰ" : "F",
|
||||
"r" : "R",
|
||||
"s" : "S",
|
||||
"t" : "S",
|
||||
"tʰ" : "TH",
|
||||
"t̠" : "L",
|
||||
"u" : "O",
|
||||
"uː" : "O",
|
||||
"v" : "F",
|
||||
"w" : "W",
|
||||
"x" : "S",
|
||||
"z" : "S",
|
||||
"æ" : "A",
|
||||
"ð" : "TH",
|
||||
"øː" : "O",
|
||||
"ŋ" : "N",
|
||||
"ɐ" : "O",
|
||||
"ɐː" : "O",
|
||||
"ɑ" : "O",
|
||||
"ɑː" : "O",
|
||||
"ɒ" : "A",
|
||||
"ɒː" : "A",
|
||||
"ɔ" : "O",
|
||||
"ɔː" : "O",
|
||||
"ɘ" : "E",
|
||||
"ə" : "E",
|
||||
"əː" : "E",
|
||||
"ɛ" : "E",
|
||||
"ɛː" : "E",
|
||||
"ɜː" : "E",
|
||||
"ɡ" : "O",
|
||||
"ɪ" : "A",
|
||||
"ɪ̯" : "A",
|
||||
"ɯ" : "E",
|
||||
"ɵː" : "A",
|
||||
"ɹ" : "R",
|
||||
"ɻ" : "R",
|
||||
"ʃ" : "S",
|
||||
"ʉ" : "O",
|
||||
"ʉː" : "O",
|
||||
"ʊ" : "O",
|
||||
"ʌ" : "O",
|
||||
"ʍ" : "W",
|
||||
"ʒ" : "S",
|
||||
"ʔ" : "O",
|
||||
"θ" : "TH",
|
||||
"d͡ʒ" : "L"
|
||||
}
|
||||
|
||||
class Lipsync():
|
||||
|
||||
def get_pose_from_phone(self, phone):
|
||||
return phones[phone]
|
||||
|
||||
def get_phones(self, syncfile):
|
||||
with open(syncfile, "r") as f:
|
||||
result = []
|
||||
for line in f.readlines():
|
||||
data = line.split()
|
||||
phone = data[2]
|
||||
start = float(data[0])
|
||||
duration = float(data[1])
|
||||
result.append((phone, start, duration))
|
||||
return result
|
||||
|
||||
def get_poses(self, syncfile):
|
||||
framerate = bpy.context.scene.render.fps
|
||||
allophones = self.get_phones(syncfile)
|
||||
result = []
|
||||
for p in allophones:
|
||||
pose = self.get_pose_from_phone(p[0])
|
||||
frame = round(p[1] / (1/framerate))
|
||||
result.append((pose, frame, p[2]))
|
||||
return result
|
||||
|
||||
def get_pose_index(self, name):
|
||||
for i, pose in enumerate([pose.name for pose in bpy.context.object.pose_library.pose_markers]):
|
||||
if name == pose:
|
||||
return i
|
||||
|
||||
def apply_pose(self, name):
|
||||
idx = self.get_pose_index(name)
|
||||
bpy.ops.poselib.apply_pose(pose_index=idx)
|
||||
[b.keyframe_insert("location") for b in bpy.context.selected_pose_bones]
|
||||
|
||||
def set_frame(self, frame):
|
||||
bpy.context.scene.frame_set(frame)
|
||||
|
||||
def apply_pose_data(self, data):
|
||||
pose = data[0]
|
||||
frame = data[1]
|
||||
self.set_frame(frame+bpy.context.object.frame_start)
|
||||
self.apply_pose(pose)
|
||||
|
||||
def apply_lipsync(self, lipsync):
|
||||
frame = bpy.data.scenes[0].frame_current
|
||||
base_pose = bpy.context.object.base_pose
|
||||
self.apply_pose_data((base_pose, lipsync[0][1]-1, 0))
|
||||
for i, pose in enumerate(lipsync):
|
||||
self.apply_pose_data(pose)
|
||||
if i < len(lipsync)-1:
|
||||
if lipsync[i+1][1]-pose[1] > 4:
|
||||
self.apply_pose_data((base_pose, i+2, 0))
|
||||
self.apply_pose_data((base_pose, lipsync[i+1][1]-1, 0))
|
||||
else:
|
||||
self.apply_pose_data((base_pose, pose[1]+3, 0))
|
||||
self.set_frame(frame)
|
||||
|
||||
# Much of the following code was taken from various blogs and Stack Overflow tutorials
|
||||
|
||||
class ApplyLipsyncing(Operator):
|
||||
"""Applies lipsyncing data"""
|
||||
bl_idname = "object.apply_lipsyncing"
|
||||
bl_label = "Apply Lipsyncing"
|
||||
|
||||
def execute(self, context):
|
||||
sync = context.object.sync_file
|
||||
test = Lipsync()
|
||||
poses = test.get_poses(sync)
|
||||
test.apply_lipsync(poses)
|
||||
return {'FINISHED'}
|
||||
|
||||
class ProcessAudio(Operator):
|
||||
"""Processes lipsync audio"""
|
||||
bl_idname = "object.process_audio"
|
||||
bl_label = "Process Audio"
|
||||
|
||||
def execute(self, context):
|
||||
if context.object.wav_file == "":
|
||||
return {'FINISHED'}
|
||||
#insert Allosaurus processing here
|
||||
result = [('F', 16, 0.025), ('O', 18, 0.025), ('R', 21, 0.025), ('S', 22, 0.025), ('E', 23, 0.025), ('L', 24, 0.025), ('S', 26, 0.025), ('O', 27, 0.025), ('L', 40, 0.025), ('A', 42, 0.025), ('L', 42, 0.025), ('O', 46, 0.025), ('S', 48, 0.025), ('A', 49, 0.025), ('S', 51, 0.025), ('S', 53, 0.025), ('A', 54, 0.025), ('L', 54, 0.025), ('S', 58, 0.025), ('S', 60, 0.025), ('R', 61, 0.025), ('A', 62, 0.025), ('F', 66, 0.025), ('E', 67, 0.025), ('L', 69, 0.025)]
|
||||
context.object.phones = json.dumps(result)
|
||||
|
||||
return {'FINISHED'}
|
||||
|
||||
class OpenFile(Operator, ImportHelper):
|
||||
"""Opens a lipsync file for processing"""
|
||||
bl_idname = 'test.open_file'
|
||||
bl_label = 'Open SYNC File'
|
||||
bl_options = {'PRESET', 'UNDO'}
|
||||
|
||||
filename_ext = '.sync'
|
||||
|
||||
filter_glob: StringProperty(
|
||||
default='*.sync',
|
||||
options={'HIDDEN'}
|
||||
)
|
||||
|
||||
def execute(self, context):
|
||||
print(self.properties.filepath)
|
||||
context.object.sync_file = self.properties.filepath
|
||||
return {'FINISHED'}
|
||||
|
||||
|
||||
class LipsyncPanel(Panel):
|
||||
bl_idname = "object.custom_panel"
|
||||
bl_label = "Automated Lipsync"
|
||||
bl_space_type = "VIEW_3D"
|
||||
bl_region_type = "UI"
|
||||
bl_category = "Lipsync"
|
||||
bl_context = "posemode"
|
||||
|
||||
|
||||
@classmethod
|
||||
def poll(self,context):
|
||||
return context.object is not None
|
||||
|
||||
def draw(self, context):
|
||||
layout = self.layout
|
||||
obj = context.object
|
||||
|
||||
layout.label(text="Select the entire MecaFace rig.")
|
||||
layout.label(text="Then, select a lipsync file below.")
|
||||
|
||||
col = layout.column(align=True)
|
||||
col.operator(OpenFile.bl_idname, text="Choose lipsync file", icon="FILE_FOLDER")
|
||||
layout.separator()
|
||||
|
||||
col = layout.column(align=True)
|
||||
col.prop(obj, "frame_start")
|
||||
col.label(text="Base pose for the start and end:")
|
||||
col.prop(obj, "base_pose")
|
||||
col.operator(ApplyLipsyncing.bl_idname, text="Apply Lipsyncing", icon="FORWARD")
|
||||
|
||||
layout.separator()
|
||||
|
||||
def register():
|
||||
register_class(LipsyncPanel)
|
||||
register_class(ApplyLipsyncing)
|
||||
register_class(OpenFile)
|
||||
register_class(ProcessAudio)
|
||||
bpy.types.Object.frame_start = bpy.props.IntProperty(name="Start Frame",
|
||||
description="The frame to start the lipsync at",
|
||||
min=0,
|
||||
default=0)
|
||||
|
||||
def get_items(self, context):
|
||||
return [(pose.name, pose.name, "Use this pose as the base pose") for pose in context.object.pose_library.pose_markers]
|
||||
bpy.types.Object.base_pose = bpy.props.EnumProperty(name="",
|
||||
description="The pose to use at the start, end, and between words",
|
||||
options={'ANIMATABLE'},
|
||||
items=get_items)
|
||||
bpy.types.Object.sync_file = bpy.props.StringProperty()
|
||||
|
||||
def unregister():
|
||||
unregister_class(LipsyncPanel)
|
||||
unregister_class(ApplyLipsyncing)
|
||||
unregister_class(OpenFile)
|
||||
unregister_class(ProcessAudio)
|
||||
del bpy.types.Object.frame_start
|
||||
del bpy.types.Object.base_pose
|
||||
del bpy.types.Object.sync_file
|
||||
|
||||
if __name__ == "__main__":
|
||||
register()
|
||||
Loading…
Reference in New Issue