binarybrat · February 14, 2020 01:24 · Feb 14, 2020
diff --git a/commentmodel.py b/commentmodel.py
@@ -0,0 +1,327 @@
+import tldextract
+from datetime import datetime, timedelta
+from models import BaseModel
+from peewee import *
+from models.blah import YoutubeLogging, Tutorials
+import time
+from reddit.utils.globalvars import *
+from reddit.utils.randomhelpershit import *
+from reddit.utils._company_rgx import *
+from models.detailsmodel import *
+from reddit.tubeChecking import *
+
+
+tutorialsfromcomments = []
+tubelinks = []
+
+
+class Comment(BaseModel):
+    approved_by = TextField(null=True)
+    archived = IntegerField(null=True)
+    author = CharField()
+    banned_by = TextField(null=True)
+    body = TextField(null=True)
+    created = DateTimeField()
+    edited = DateTimeField(null=True)
+    distinguished = CharField(null=True)
+    posflair = CharField(null=True)
+    fullname = CharField()
+    gilded = CharField(null=True)
+    idstr = CharField(unique=True)
+    ignore_reports = IntegerField(null=True)
+    is_root = IntegerField()
+    is_submitter = IntegerField()
+    link_author = CharField(null=True)
+    link_title = TextField(null=True)
+    link_permalink = TextField(null=True)
+    link_url = TextField(null=True)
+    spam = CharField(null=True)
+    submission = CharField(null=True)
+    parent = CharField(null=True)
+    permalink = TextField(null=True)
+    removed = IntegerField(null=True)
+    score = IntegerField(null=True)
+    stickied = IntegerField()
+    subreddit = CharField(null=True)
+    ups = IntegerField()
+    perspective = TextField(null=True)
+    ureports_dismissed = TextField(null=True)
+    fetched = DateTimeField(constraints=[SQL("DEFAULT CURRENT_TIMESTAMP")], null=True)
+
+    class Meta:
+        table_name = 'Comment'
+
+
+class CommentEdits(BaseModel):
+    idstr = CharField()
+    text = TextField(null=True)
+    replaced_at = DateTimeField()
+    subreddit = CharField()
+
+    class Meta:
+        table_name = 'CommentEdits'
+        indexes = (
+            (('idstr', 'replaced_at'), True),
+        )
+        primary_key = CompositeKey('idstr', 'replaced_at')
+
+
+def lastComment():
+    commentsDB = Comment.select()
+    test = [pos.created for pos in commentsDB.order_by(Comment.created.desc()).limit(1)]
+    return time.mktime(test[0].timetuple())
+
+
+def cleancomlinks(lisofcomlinks):
+    newlinklist = []
+    for comlink in lisofcomlinks:
+        link = comlink.split("</a>")[0] if "</a>" in comlink else comlink
+        if link not in newlinklist:
+            newlinklist.append(link)
+
+    return newlinklist
+
+
+def comDone(commentobj):
+    comdone = Comment.get_or_none(Comment.fullname == commentobj)
+    return True if comdone else False
+
+
+class CommentHelper:
+    def __init__(self, reddit, comment, grablinks=False, dammit=False, verbose=False, overkill=False, wordmin=50):
+        self.verbose = verbose
+        self.overkill = overkill
+        self.grablinks = grablinks
+        self.reddit = reddit
+        self.comment = comment
+        self.permalink = 'https://www.reddit.com' + self.comment.permalink
+        self.wordmin = wordmin
+        self.wordcount = len(self.comment.body.split())
+        self.submission = self.reddit.submission(self.comment.submission.id)
+        self.link_author = self.submission.author if self.submission.author is not None else '[deleted]'
+        self.link_title = self.submission.title
+        self.link_permalink = 'https://www.reddit.com' + self.submission.permalink
+        self.link_url = self.submission.url
+        self.link_score = self.submission.score
+        self.link_flair = self.submission.link_flair_text if self.submission.link_flair_text is not None else None
+        self.is_deleted = True if (self.comment.author is None and self.comment.body == '[deleted]') else False
+        self.submission_deleted = True if self.submission.author is None or self.comment.body == '[deleted]' else False
+        self.pattern_comment = True if (
+                re.search(bcomprgx, self.comment.body) or re.search(newergx, self.comment.body) or re.search(
+            burdargx, self.comment.body)) else False
+        self.tutorial_comment = True if (re.search(r"(tutorial[s]?|free pattern[s]?)", self.comment.body,
+                                                   re.IGNORECASE) and 'http' in self.comment.body.lower()) else False
+        self.details_comment = True if not isdeleted(self.comment) and not self.comment.spam and not self.comment.removed and not isdeleted(
+            self.submission) and not self.submission.spam and not (self.submission.removed and self.submission.banned_by is not None and self.submission.banned_by != 'sewingmodthings') and self.comment.is_submitter and self.comment.parent_id.startswith('t3_') and self.link_flair is not None and self.link_flair.lower() in [x.lower() for x in
+                                                                                   FLAIRTOCHECK] and (
+                                                   self.pattern_comment or self.wordcount >= self.wordmin) else False
+        self.is_rising = True if self.submission.score >= 900 else False
+        self.previewimg = self.submission.preview['images'][0]['resolutions'][2]['url'] if self.details_comment and has_preview(self.submission) else None
+        self.has_comlinks = False
+        self.low_comment = False
+        self.dammit = dammit
+
+        if self.dammit:
+            self.doitdammit()
+
+        if self.grablinks and not isdeleted(
+                self.comment) and not self.comment.spam and not self.comment.removed:
+            self.grab_commentlinks()
+            if self.commentlinks:
+                self.has_comlinks = True
+
+    def doitdammit(self):
+        self.existing = Comment.get_or_none(Comment.idstr == self.comment.id)
+        if not self.existing:
+            self.add_comment()
+        else:
+            if self.comment.author is None and self.comment.body == '[deleted]':
+                self.updateDeleted_comment()
+            else:
+                self.update_comment()
+
+    def add_comment(self):
+        Comment.create(
+            approved_by=self.comment.approved_by if self.comment.approved_by is None else None,
+            archived=self.comment.archived,
+            author=self.comment.author.name if self.comment.author else "[deleted]",
+            banned_by=self.comment.banned_by if self.comment.banned_by is None else None,
+            created=datetime.fromtimestamp(self.comment.created_utc),
+            distinguished=self.comment.distinguished,
+            fullname=self.comment.fullname,
+            gilded=self.comment.gilded,
+            idstr=self.comment.id,
+            posflair=self.link_flair,
+            is_root=self.comment.is_root,
+            is_submitter=self.comment.is_submitter,
+            link_author=self.link_author,
+            link_title=self.link_title,
+            link_permalink=self.link_permalink,
+            link_url=self.link_url,
+            removed=self.comment.removed,
+            spam=self.comment.spam,
+            submission=self.comment.submission.id,
+            parent=None if self.comment.is_root else self.comment.parent_id[3:],
+            permalink=self.comment.permalink,
+            subreddit=self.comment.subreddit.display_name,
+            perspective=None,
+            body=self.comment.body,
+            edited=datetime.fromtimestamp(self.comment.edited) if self.comment.edited else None,
+            score=self.comment.score,
+            stickied=self.comment.stickied,
+            ups=self.comment.ups,
+            ureports_dismissed=self.comment.user_reports_dismissed if hasattr(self.comment,
+                                                                              'user_reports_dismissed') else None
+        )
+        if self.verbose:
+            print("Added comment: {} by u/{}".format(self.comment.fullname, self.comment.author.name if
+            self.comment.author is not None else '[deleted]'))
+
+    def update_comment(self):
+        toupdate = Comment.update(
+            approved_by=self.comment.approved_by if self.comment.approved_by is not None else None,
+            archived=self.comment.archived,
+            banned_by=self.comment.banned_by,
+            body=self.comment.body,
+            edited=datetime.fromtimestamp(self.comment.edited) if self.comment.edited else None,
+            distinguished=self.comment.distinguished,
+            posflair=self.link_flair,
+            gilded=self.comment.gilded,
+            ignore_reports=self.comment.ignore_reports,
+            removed=self.comment.removed,
+            spam=self.comment.spam,
+            score=self.comment.score,
+            stickied=self.comment.stickied,
+            ups=self.comment.ups,
+            ureports_dismissed=self.comment.user_reports_dismissed if hasattr(self.comment, 'user_reports_dismissed') else None).where(Comment.idstr == self.comment.id)
+        toupdate.execute()
+        if self.overkill:
+            return print("Updated comment: {}".format(self.comment.fullname))
+
+    def updateDeleted_comment(self):
+        todelupdate = Comment.update(
+            approved_by=self.comment.approved_by if self.comment.approved_by is not None else None,
+            archived=self.comment.archived,
+            banned_by=self.comment.banned_by,
+            edited=datetime.fromtimestamp(self.comment.edited) if self.comment.edited else None,
+            distinguished=self.comment.distinguished,
+            posflair=self.link_flair,
+            gilded=self.comment.gilded,
+            ignore_reports=self.comment.ignore_reports,
+            removed=self.comment.removed,
+            spam=self.comment.spam,
+            score=self.comment.score,
+            stickied=self.comment.stickied,
+            ups=self.comment.ups,
+            ureports_dismissed=self.comment.user_reports_dismissed if
+            hasattr(self.comment, 'user_reports_dismissed') else None).where(Comment.idstr == self.comment.id)
+        todelupdate.execute()
+        if self.verbose:
+            print("Updated DELETED comment: {}".format(self.comment.fullname))
+
+    def insert_edited(self, old_text):
+        if self.comment.edited is False:
+            replaced_at = datetime.now()
+        else:
+            replaced_at = datetime.fromtimestamp(self.comment.edited)
+        try:
+            CommentEdits.create(idstr=self.comment.fullname, replaced_at=replaced_at, text=old_text,
+                                subreddit=self.comment.subreddit.display_name)
+            if self.verbose:
+                return print("Inserted Edited Comment For: {}".format(self.comment.fullname))
+        except IntegrityError:
+            pass
+
+    def already_in_db(self):
+        self.cdone = Comment.get_or_none(Comment.idstr == self.comment.id)
+        return True if self.cdone is not None else False
+
+    def check_for_pattern(self):
+        if re.search(burdargx, self.comment.body) or re.search(bcomprgx, self.comment.body) or re.search(newergx,
+                                                                                                         self.comment.body):
+            self.pattern_comment = True
+
+    def check_for_tutorial(self):
+        if re.search(r"(tutorial[s]?|free pattern[s]?)", self.comment.body, re.IGNORECASE):
+            self.tutorial_comment = True
+
+    def check_for_details(self):
+        if not isdeleted(self.comment) and not self.comment.spam and not self.comment.removed:
+            if self.comment.is_submitter and self.comment.parent_id.startswith(
+                    't3_') and self.link_flair is not None and self.link_flair.lower() in [x.lower() for x in
+                                                                                           FLAIRTOCHECK]:
+                if self.pattern_comment or len(self.comment.body.split()) >= self.wordmin:
+                    self.createdat = datetime.utcfromtimestamp(self.comment.created_utc)
+                    self.details_comment = True
+
+    def grab_commentlinks(self):
+        self.commentlinks = findhtml(self.comment.body_html)
+        if self.commentlinks:
+            self.has_comlinks = True
+
+    def low_comment_check(self):
+        if self.comment.score <= -5 and self.comment.approved_by is None and self.comment.banned_by is None and not \
+                self.comment.removed and self.comment.author is not None:
+            self.low_comment = True
+
+    def force_details(self):
+        fupdate = DetailsPending.update(ignore=True).where(DetailsPending.idstr == self.submission.fullname)
+        fupdate.execute()
+        print("Forced Pending Details to approved since a mod already approved the post.")
+
+    def approve_details_silent(self):
+        return True if time.time() <= 1565762849.0 else False
+
+
+def grab_author_backcoms(reddit, authorname, verbose=False):
+    count = 0
+    processed = 0
+    delupdated = 0
+    updated = 0
+
+    for comment in reddit.redditor(authorname).comments.new(limit=None):
+        count += 1
+        if count % 300 == 0:
+            print(count, 'done')
+        if comment.subreddit.display_name == 'sewing' and not isdeleted(comment):
+            try:
+                chelper = CommentHelper(reddit, comment, grablinks=True, verbose=verbose)
+                if not chelper.already_in_db():
+                    chelper.add_comment()
+                    processed += 1
+                    if chelper.has_comlinks and chelper.comment.author != 'sewingmodthings' and (
+                            chelper.link_flair is None or chelper.link_flair is not None and chelper.link_flair.lower() not in
+                            ['machine questions', 'suggest machine', 'fabric question']):
+                        commentlinks = cleancomlinks(chelper.commentlinks)
+                        for linkurl in commentlinks:
+                            if re.search(tubergx, linkurl) and ("/channel/" or "/user/") not in linkurl:
+                                process_tube_data(chelper.comment, linkurl)
+                                time.sleep(5)
+                            else:
+                                ext = tldextract.extract(linkurl)
+                                if not tutDone(chelper.comment.fullname, linkurl) and not (
+                                        ext.domain == 'spoonflower' and ext.subdomain == 'www') and not re.search(
+                                        parserdomainsrgx, ext.domain) and not re.search(domainstoignore,
+                                                                                        ext.domain) and not re.search(
+                                        isfilergx, linkurl):
+                                    try:
+                                        linktitle = grab_tutorial_info(chelper.comment.fullname, linkurl, verbose=True)
+                                        add_tutorial(chelper.comment, tags=None, linktitle=linktitle, linkurl=linkurl,
+                                                     verbose=True)
+                                        time.sleep(5)
+
+                                    except IntegrityError:
+                                        pass
+                                    except Exception as e:
+                                        pass
+                else:
+                    if chelper.is_deleted:
+                        chelper.updateDeleted_comment()
+                        delupdated += 1
+                    else:
+                        chelper.update_comment()
+                        updated += 1
+            except IndexError:
+                pass
+
+    print("Comments >> Processed: {}  ||  Updated: {}".format(processed, updated))