Merge pull request #6 from arehbein-git/#5-implement-date-based-fetching-and-processing

#5: add date based process, fix paths (win/linux)

Author: arehbein-git

photoElement.py

@@ -0,0 +1,32 @@
+from datetime import datetime, date
+
+class PhotoElement():
+    """A Object holding the information of the photo"""
+
+    _path = ""
+    _date = datetime.today().date()
+    _tags = []
+    _rating = 0
+
+    def __init__(self, path, date, tags, rating):
+        """path: path of photo including filename, date: date photo was taken [datetime.date], tags: xmp tags [list], rating: 0-5"""
+        self._path = path
+        self._date = date
+        self._tags = tags
+        self._rating = rating
+    
+    def path(self):
+        """returns the path of the photo"""
+        return self._path
+    
+    def date(self):
+        """returns the date of the photo"""
+        return self._date
+    
+    def tags(self):
+        """returns the tags of the photo"""
+        return self._tags
+
+    def rating(self):
+        """returns the rating of the photo"""
+        return self._rating

pptag.py

@@ -8,6 +8,8 @@
 import logging
 import urllib
 import time
+import os
+from datetime import datetime, date, timedelta
 from threading import Timer
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
@@ -17,6 +19,7 @@
 from OneShotQueueTimer import OneShotQueueTimer
 from plexUsers import plexUsers
 from lightroomTags import parse_xmp_for_lightroom_tags
+from photoElement import PhotoElement
 
 from config import ppTagConfig
 
@@ -124,6 +127,8 @@ def updateTagsAndRating(key, filename):
         # get the tags
         data = process_file(img_file, stop_tag=stop_tag, details=detailed, strict=strict, debug=debug)
 
+        img_file.close()
+
         if not data:
             #print("No EXIF information found\n")
             return
@@ -152,8 +157,68 @@ def updateTagsAndRating(key, filename):
         # it is a corrupt file (exif/xmp)
         return
 
+def parseExifAndTags(filename):
+
+    detailed = True
+    stop_tag = DEFAULT_STOP_TAG
+    debug = False
+    strict = False
+    color = False
+
+    #exif_log.setup_logger(debug, color)
+
+    filepath = ppTagConfig.PHOTOS_LIBRARY_PATH + filename
+
+    try:
+        img_file = open(str(filepath), 'rb')
+    except IOError:
+        # print("'%s' is unreadable" % filename)
+        return None
+
+    try:
+        # get the tags
+        data = process_file(img_file, stop_tag=stop_tag, details=detailed, strict=strict, debug=debug)
+
+        img_file.close()
+
+        if not data:
+            #print("No EXIF information found\n")
+            return None
+
+        if 'JPEGThumbnail' in data:
+            # logger.info('File has JPEG thumbnail')
+            del data['JPEGThumbnail']
+        if 'TIFFThumbnail' in data:
+            # logger.info('File has TIFF thumbnail')
+            del data['TIFFThumbnail']
 
-def triggerLoopThroughAllPhotos():
+        parsedXMP = {}
+        parsedXMP['rating'] = 0
+        parsedXMP['tags'] = []
+        # xmp data
+        if 'Image ApplicationNotes' in data:
+            xml = data['Image ApplicationNotes'].printable
+
+            parsedXMP = parse_xmp_for_lightroom_tags(xml)
+
+        # if 'Image Copyright' in data:
+        #     print("Copyright : %s", data['Image Copyright'].printable)
+
+        date = datetime.today().date()
+        if 'EXIF DateTimeOriginal' in data:
+            date = datetime.strptime(data['EXIF DateTimeOriginal'].printable, '%Y:%m:%d %H:%M:%S').date()
+        else:
+            datetimeModified = datetime.fromtimestamp(os.path.getmtime(filepath))
+            date = datetimeModified.date()
+        
+        photoElement = PhotoElement(filename, date, parsedXMP['tags'], parsedXMP['rating'])
+        return photoElement
+    except:
+        # it is a corrupt file (exif/xmp)
+        return None
+
+
+def triggerProcess():
     global t
     t.start()
 
@@ -162,8 +227,91 @@ def uniqify(seq):
     keys = {}
     for e in seq:
         keys[e] = 1
-    return keys.keys()
+    return list(keys.keys())
 
+def fetchPhotosAndProcess():
+    global firstRun
+
+    if firstRun:
+        # if a complete update on startup is requested loop through all photos
+        loopThroughAllPhotos()
+    else:
+        # else fetch all photos based on date
+        if fetchAndProcessByDate():
+            # failed so loop through all photos
+            loopThroughAllPhotos()
+
+def fetchAndProcessByDate():
+    global doUpdate
+    doUpdateTemp = uniqify(doUpdate)
+    doUpdate = []
+
+    photoGroups = {}
+    # first group all photos by date
+    for filepath in doUpdateTemp:
+        photoElement = parseExifAndTags(filepath)
+        if photoElement:
+            # this has exif data
+            date = photoElement.date()
+            if date in photoGroups.keys():
+                photoGroups[date].append(photoElement)
+            else:
+                photoGroups[date] = [photoElement]
+
+    for date in photoGroups.keys():
+        #print(date)
+        fromTimecode = int(datetime.strptime(date.isoformat(), '%Y-%m-%d').timestamp())
+        toTimecode = int((datetime.strptime(date.isoformat(), '%Y-%m-%d') + timedelta(days=1)).timestamp())-1
+
+        toDo = True
+        start = 0
+        size = 1000
+
+        plexData = {}
+        #print('loop through all, started %i' % int(time.time()))
+        while toDo:
+            if len(p.photoSections):
+                url = "/library/sections/" + p.photoSections[0] + "/all?originallyAvailableAt%3E=" + str(fromTimecode) + "&originallyAvailableAt%3C=" + str(toTimecode) + "&X-Plex-Container-Start=%i&X-Plex-Container-Size=%i" % (start, size)
+                metadata = p.fetchPlexApi(url)
+                container = metadata["MediaContainer"]
+                elements = container["Metadata"]
+                totalSize = container["totalSize"]
+                offset = container["offset"]
+                size = container["size"]
+                start = start + size
+                if totalSize-offset-size == 0:
+                    toDo = False
+                # loop through all elements
+                for photo in elements:
+                    mediaType = photo["type"]
+                    if mediaType != "photo":
+                        continue
+                    key = photo["ratingKey"]
+                    src = photo["Media"][0]["Part"][0]["file"].replace(ppTagConfig.PHOTOS_LIBRARY_PATH_PLEX,"", 1)
+
+                    plexData[src] = key
+
+        for photo in photoGroups[date]:
+            path = photo.path()
+            # make sure path seperator is equal in plex and ppTag
+            if "/" in ppTagConfig.PHOTOS_LIBRARY_PATH_PLEX:
+                path = path.replace("\\","/")
+            if path in plexData.keys():
+                updateMetadata(plexData[path], photo.tags(), photo.rating()*2)
+                photoGroups[date].remove(photo)
+        
+        
+        for photo in photoGroups[date]:
+            # we need to fetch all data as this method failed
+            # print(photo.path() + " was not processed!")
+            doUpdate = [*doUpdate, *doUpdateTemp]
+            return True
+    
+    # after the loop we maybe have new or modifed files which was blocked before so trigger again
+    if len(doUpdate):
+        triggerProcess()
+    
+    return False
 
 def loopThroughAllPhotos():
     global doUpdate
@@ -172,7 +320,7 @@ def loopThroughAllPhotos():
     doUpdate = []
     toDo = True
     start = 0
-    size = 100
+    size = 1000
     #print('loop through all, started %i' % int(time.time()))
     while toDo:
         if len(p.photoSections):
@@ -194,17 +342,30 @@ def loopThroughAllPhotos():
                 key = photo["ratingKey"]
                 src = photo["Media"][0]["Part"][0]["file"].replace(ppTagConfig.PHOTOS_LIBRARY_PATH_PLEX,"", 1)
 
+                # make sure path seperator is equal in plex and ppTag
+                if "\\" in ppTagConfig.PHOTOS_LIBRARY_PATH:
+                    src = src.replace("/","\\")
+
                 if src in doUpdateTemp or firstRun:
 
                     # update tags and rating
                     # print(key)
                     # print(src)
                     updateTagsAndRating(key, src)
+                    if not firstRun:
+                        doUpdateTemp.remove(src)
+                
+                if len(doUpdateTemp) == 0 and not firstRun:
+                    # finished
+                    # after the loop we maybe have new or modifed files which was blocked before so trigger again
+                    if len(doUpdate):
+                        triggerProcess()
+                    return
 
 
     # after the loop we maybe have new or modifed files which was blocked before so trigger again
     if len(doUpdate):
-        triggerLoopThroughAllPhotos()
+        triggerProcess()
         #print("change detected while processing, retrigger")
     #print('loop through all, done %i' % int(time.time()))
     firstRun = False
@@ -227,7 +388,7 @@ def process(self, event):
             if not event.is_directory:
                 # put file into forced update list
                 doUpdate.append(event.src_path.replace(ppTagConfig.PHOTOS_LIBRARY_PATH,"", 1))
-                triggerLoopThroughAllPhotos()
+                triggerProcess()
 
     def on_modified(self, event):
         self.process(event)
@@ -244,12 +405,12 @@ def on_created(self, event):
 
     # setup timer
     # wait 120 sec after change was detected
-    t = OneShotQueueTimer(120, loopThroughAllPhotos)
+    t = OneShotQueueTimer(120, fetchPhotosAndProcess)
 
     p = plexUsers()
 
     # run at startup
-    loopThroughAllPhotos()
+    fetchPhotosAndProcess()
 
     # now start the observer
     observer.start()
@@ -261,4 +422,3 @@ def on_created(self, event):
         observer.stop()
 
     observer.join()
-