Skip to content

Commit

Permalink
Merge pull request #185 from kieranjol/copyit
Browse files Browse the repository at this point in the history 
Copyit
  • Loading branch information
@kieranjol
kieranjol authored Jul 12, 2017
2 parents 388d9c3 + eba9a5c commit 2bc3645
Show file tree
Hide file tree
Showing 8 changed files with 1,249 additions and 452 deletions.
16 changes: 9 additions & 7 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,12 @@ Note: Documentation template has been copied from [mediamicroservices](https://g
## Arrangement ##

### sipcreator.py ###
* Accepts one or more directories as input and wraps them up in a directory structure in line with IFI procedures using `moveit.py`.
* Folders will be stored in an objects directory. Directory structure is a parent directory named with a UUID, with three child directories (objects, logs metadata):
* Checksums are stored for the package and metadata is extracted for the AV material in the objects. A log records the major events in the process.
* Accepts one or more files or directories as input and wraps them up in a directory structure in line with IFI procedures using `copyit.py`.
* Source objects will be stored in an /objects directory. Directory structure is: parent directory named with a UUID, with three child directories (objects, logs metadata):
* Metadata is extracted for the AV material and MD5 checksums are stored for the entire package. A log records the major events in the process.
* Usage for one directory - `sipcreator.py -i /path/to/directory_name -o /path/to/output_folder`
* Usage for more than one directory - `sipcreator.py -i /path/to/directory_name1 /path/to/directory_name2 -o /path/to/output_folder`
* Run `sipcreator.py -h` for all options.

## Transcodes ##

Expand All @@ -87,10 +88,11 @@ Note: Documentation template has been copied from [mediamicroservices](https://g
* This script has many extra options, such as deinterlacing, quality settings, rescaling. Use `prores.py -h` to see all options

### concat.py ###
* Concatenate/join video files together using ffmpeg stream copy into a single Matroska container. As the streams are losslessly copied, the speed is quite fast.
* Concatenate/join video files together using ffmpeg stream copy into a single Matroska container. Each source clip will have its own chapter marker. As the streams are copied, the speed is quite fast.
* Usage: `concat.py -i /path/to/filename1.mov /path/to/filename2.mov -o /path/to/destination_folder`
* A lossless verification process will also run, which takes stream level checksums of all streams and compares the values. This is not very reliable at the moment.
* Warning - video files must have the same technical attributes such as codec, width, height, fps. Some characters in filenames will cause the script to fail. Some of these include quotes. The script will ask the user if quotes should be renamed with underscores. Also, a temporary concatenation textfile will be stored in your temp folder. Currently only tested on Ubuntu.

* Dependencies: mkvpropedit, ffmpeg.
## Digital Cinema Package Scripts ##

### dcpaccess.py ###
Expand All @@ -112,8 +114,8 @@ Note: Documentation template has been copied from [mediamicroservices](https://g

## Fixity Scripts ##

### moveit.py ###
* Copies a directory, creating a md5 manifest at source and destination and comparing the two. Skips hidden files and directories.
### copyit.py ###
* Copies a file or directory, creating a md5 manifest at source and destination and comparing the two. Skips hidden files and directories.
* Usage: ` moveit.py source_dir destination_dir`
* Dependencies: OSX requires gcp - `brew install coreutils`

Expand Down
155 changes: 147 additions & 8 deletions concat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
import subprocess
import os
import argparse
import time
import shutil
import sipcreator
import ififuncs


def parse_args(args_):
'''
Parse command line arguments.
Expand All @@ -29,6 +31,17 @@ def parse_args(args_):
'-r', '-recursive',
help='recursively process all files in subdirectories. This could be potentially a disaster - so use with caution or with XDCAM', action='store_true'
)
parser.add_argument(
'--no-sip',
help='Do not run sipcreator.py on the resulting file.', action='store_true'
)
parser.add_argument(
'-user',
help='Declare who you are. If this is not set, you will be prompted.')
parser.add_argument(
'-oe',
help='Enter the Object Entry number for the representation.SIP will be placed in a folder with this name.'
)
parsed_args = parser.parse_args(args_)
return parsed_args

Expand All @@ -37,17 +50,22 @@ def ffmpeg_concat(concat_file, args, uuid):
'''
Launch the actual ffmpeg concatenation command
'''
fmd5_logfile = os.path.join(args.o, '%s_concat.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
fmd5_env_dict = ififuncs.set_environment(fmd5_logfile)
print fmd5_logfile
print fmd5_env_dict
cmd = [
'ffmpeg', '-f', 'concat', '-safe', '0',
'ffmpeg', '-report', '-f', 'concat', '-safe', '0',
'-i', concat_file,
'-c', 'copy', '-map', '0:a?', '-map', '0:v',
os.path.join(args.o, '%s.mkv' % uuid)
'-c', 'copy', '-map', '0:v', '-map', '0:a?',
os.path.join(args.o, '%s.mkv' % uuid),
'-f', 'md5', '-map', '0:v', '-map', '0:a?','-c', 'copy', '-'
]
print cmd
subprocess.call(
cmd
source_bitstream_md5 = subprocess.check_output(
cmd, env=fmd5_env_dict
)

return source_bitstream_md5.rstrip(), fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')

def recursive_file_list(video_files):
'''
Expand All @@ -67,6 +85,25 @@ def recursive_file_list(video_files):
print recursive_list
return recursive_list

def make_chapters(video_files):
millis = ififuncs.get_milliseconds(video_files[0])
timestamp = ififuncs.convert_millis(int(millis))
chapter_list = [['00:00:00.000', os.path.basename(video_files[0])], [timestamp, os.path.basename(video_files[1])]]
count = 2
for video in video_files[1:]:
millis += ififuncs.get_milliseconds(video)
timestamp = ififuncs.convert_millis(int(millis))
if count == len(video_files):
continue
else:
chapter_list.append([timestamp, os.path.basename(video_files[count])])
count+=1
chapter_counter = 1
# uh use a real path/filename.
with open('chapters.txt', 'wb') as fo:
for i in chapter_list:
fo.write('CHAPTER%s=%s\nCHAPTER%sNAME=%s\n' % (str(chapter_counter).zfill(2), i[0], str(chapter_counter).zfill(2), i[1]))
chapter_counter += 1


def main(args_):
Expand All @@ -75,13 +112,115 @@ def main(args_):
'''
uuid = ififuncs.create_uuid()
args = parse_args(args_)
print args
log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
ififuncs.generate_log(log_name_source, 'concat.py started.')
ififuncs.generate_log(
log_name_source,
'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
ififuncs.generate_log(
log_name_source,
'Command line arguments: %s' % args
)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) != 4:
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
ififuncs.generate_log(
log_name_source,
'EVENT = agentName=%s' % user
)
source_uuid_check = ififuncs.check_for_uuid(args)
if source_uuid_check == False:
source_uuid = ififuncs.get_source_uuid()
else: source_uuid = source_uuid_check
ififuncs.generate_log(
log_name_source,
'Relationship, derivation, has source=%s' % source_uuid
)
video_files = args.i
concat_file = ififuncs.get_temp_concat('concat_stuff')
ififuncs.generate_log(
log_name_source,
'concatenation file=%s' % concat_file)
if args.r:
video_files = recursive_file_list(video_files)
video_files = ififuncs.sanitise_filenames(video_files)
for source_files in video_files:
ififuncs.generate_log(
log_name_source,
'source_files = %s' % source_files)
make_chapters(video_files)
ififuncs.concat_textfile(video_files, concat_file)
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid)
output_file = os.path.join(args.o, '%s.mkv' % uuid)
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
validation_env_dict = ififuncs.set_environment(validation_logfile)
output_bitstream_md5 = subprocess.check_output([
'ffmpeg', '-report',
'-i', output_file,
'-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
], env=validation_env_dict).rstrip()
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
if source_bitstream_md5 == output_bitstream_md5:
print 'process appears to be lossless'
print source_bitstream_md5, output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=pass')
else:
print 'something went wrong - not lossless!'
print source_bitstream_md5,output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=fail')
subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
ififuncs.generate_log(
log_name_source,
'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
ififuncs.concat_textfile(video_files, concat_file)
ffmpeg_concat(concat_file, args, uuid)
with open(log_name_source, 'r') as concat_log:
concat_lines = concat_log.readlines()
if not args.no_sip:
sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
logs_dir = os.path.dirname(sipcreator_log)
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
with open(sipcreator_log, 'r') as sipcreator_log_object:
sipcreator_lines = sipcreator_log_object.readlines()
with open(sipcreator_log, 'wb') as fo:
for lines in concat_lines:
fo.write(lines)
for remaining_lines in sipcreator_lines:
fo.write(remaining_lines)
ififuncs.checksum_replace(sipcreator_manifest, sipcreator_log)

if __name__ == '__main__':
main(sys.argv[1:])
Loading

0 comments on commit 2bc3645

Please sign in to comment.