Improved Logging. Automated DB connection and closing. Improved .conf controls for emails and live_google.

CHANGELOG

@@ -11,7 +11,7 @@ Search critiera can only be modified in the index.py code by adding criteria to
 index.py broke on multiple instances of the word 'address' on page 2 of the georgiamls listings.  Added fix to
 set the address attribute to only the first instance of the word.
 
-[0.1.1] Update .1 WIP
+[0.1.1] Update .1
 
 applied hotfix 1
 Added ConfigParser to read landsearch.conf file and gather search criteria from this file.
@@ -20,11 +20,9 @@ Added checktype to Search class.  This ensures that the word None in landsearch.
 Changed email module.  Forces email in email=True in conf file.  Sends nicely formatted email for empty results.
 Added total new properties found in email subject line.
 Improved Logging.
-    TODO - create logging class and check if instance created in each module.
-    This way
-
-FUTURE - allow logging to both logfile AND console
-FUTURE - add .conf section for results.py to aid in results report generation.
+Removed need for user to call connect_db() and close_db() in user code.
+Added live_google to .conf file to allow user to select wether or not to call Google API.
+Applied Update1 - added datestamp to logging output
 
 
 FUTURE GOALS
@@ -40,3 +38,5 @@ FUTURE GOALS
   - Show results within X time/dist to school/work
   - Show various columns in output
   - Output results to csv.
+- Allow logging to both console and logfile
+- Add sectionin .conf file for results.py report generation

index.py

@@ -16,19 +16,12 @@ import logging
 
 ### TO DO ###
 #
-#  Print useful reports (land only, house and land, etc)
 #  Check if db entries no longer appear online (mark expired)
 #  When checking online from various sites, check if address already exists in db
 #    - if so, warn user and do not add
-#  Add date_added to initial entries
 #  Check results against database for changes
 #   - update and add/change date_modified 
-#  Add argument to run update query when results.py is calles
-#  Add database column to hold parcel number.  Make links to GIS servers 
-#
-#  IDENTIFY NEW PROPERTIES!!
-#
-#  Automate db opening and closing when calling dbinsert()
+#  Add database column to hold parcel number.  Make links to GIS servers
 #
 #############
 
@@ -54,6 +47,7 @@ class Property:
         self.description = description
         self.link = link
 
+
 class Parameters:
     '''Parameters taken from config file'''
 
@@ -69,15 +63,18 @@ class Parameters:
         except Exception as err:
             print(err, "Using default search Parameters")
 
+
 class Mylogger:
     ''' Logging tool for this session'''
+
     def __init__(self):
         log_params = Parameters().log_params
 
-        filename=log_params.get('log_file')
-        level=int(log_params.get('logging_level', str('30')))
-        format='%(asctime)s %(levelname)-8s %(message)s'
-        datefmt='%Y-%m-%d %H:%M:%S'
+        filename = log_params.get('log_file')
+        level = int(log_params.get('logging_level', str('30')))
+        format = '%(asctime)s %(levelname)-8s %(message)s'
+        datefmt = '%Y-%m-%d %H:%M:%S'
+
 
 class Search:
     '''Universal Search Criteria'''
@@ -140,18 +137,20 @@ class Search:
         ## FOR TESTING, PRINT ALL ATTRIBUTES OF SEARCH ##
         logging.debug(vars(self))
 
+
 class ImproperSearchError(Exception):
     def __init__(self, search, message="Improper Search.  Must use instance of Search class"):
         self.search = search
         self.message = message
         super().__init__(self.message)
 
+
 class MLSDATA:
     """Fetches and stores MLS Data
      Currently only supports GeorgiaMLS.com (GMLS)"""
     counties = ['Gwinnett', 'Barrow', 'Hall', 'Jackson', 'Walton']
     GoogleAPIKey = 'AIzaSyAXAnpBtjv760W8YIPqKZ0dFXpwAaZN7Es'
-    live_google = True
+    # live_google = False
 
     def __init__(self, mlstype):
         self.parameters = Parameters()
@@ -161,6 +160,7 @@ class MLSDATA:
         self.cnx = ''
         self.new_listings = []
         self.email = self.parameters.search_params.getboolean('email')
+        self.live_google = self.parameters.search_params.getboolean('live_google')
         print('Email  ' + str(self.email))
 
     def stringbuilder(self, search: Search, county):
@@ -297,17 +297,25 @@ class MLSDATA:
                 print("Scanning for results in " + county + " using the " + self.mlstype.upper() + " database.")
                 if self.mlstype == 'gmls':
                     list = self.gmlsparser(self.stringbuilder(search, county), county)
-                logging.info("Completed search in " + county + " county. " + str(len(list)) + " total properties scanned.")
+                logging.info(
+                    "Completed search in " + county + " county. " + str(len(list)) + " total properties scanned.")
                 return list
         else:
             raise ImproperSearchError(search)
 
-    def checkdb(self, criteria_dict):
+    def check_db(self, criteria_dict):
         """Check dictionary of critera against database.
        Currently accepts keys: MLS, title, address (street number/name, not city/state/zip).
        Returns True if records exists."""
-        if self.cursor:  ## Check if DB is connected
-            for criteria in criteria_dict:
+        if not self.cursor:  ## Check if DB is connected
+            try:
+                self.connect_db()
+                logging.debug("No Database Connection.  Connecting to DB in check_db function.")
+            except Exception as err:
+                print("Could not connect to Database. " + str(err))
+                logging.warning("Could not connect to Database. " + str(err))
+                return 0
+        for criteria in criteria_dict:
                 ## Determine criteria passed, and execute queries for each
                 if criteria == 'MLS':
                     self.cursor.execute("SELECT COUNT(*) FROM properties WHERE MLS = %(MLS)s GROUP BY id",
@@ -323,11 +331,9 @@ class MLSDATA:
                     if self.cursor.rowcount > 0: return self.cursor.rowcount  # stop for loop if match already found.
                 else:
                     print("Cannot search on parameter: " + criteria)
-            return self.cursor.rowcount
-        else:
-            print("Database is not connected or cursor not filled.  Use function 'connectdb()' to establish")
+        return self.cursor.rowcount
 
-    def getGoogle(self, property):
+    def get_google(self, property):
         """Supplies date from Google Distance Matrix API to populate
       distance_to_work
       time_to_work
@@ -362,6 +368,16 @@ class MLSDATA:
 
     def insertrecord(self, property, work_address=None, school_address=None):
         """Inserts record into database.  Takes argument Property class object."""
+        if not self.cursor:
+            print("not self.cursor")
+            logging.debug("MYSQL connection not established.  Trying to connect...")
+            try:
+                self.connect_db()
+                logging.debug("Connecting to DB in insertrecord fucntion.")
+            except Exception as err:
+                print("Could not connect to Database. " + str(err))
+                logging.warning("Could not connect to Database. " + str(err))
+                return
         if self.cursor:
             criteria_dict = property.__dict__
             criteria_dict['Date_Added'] = str(datetime.date.today())
@@ -373,7 +389,7 @@ class MLSDATA:
                 self.cursor.execute(qry)
                 self.cnx.commit()
                 print("Inserted " + criteria_dict['MLS'] + " | " + criteria_dict['address'] + " into database.")
-                logging.debug("Inserted " + criteria_dict['MLS'] + " | " + criteria_dict['address'] + " into database.")
+                logging.info("Inserted " + criteria_dict['MLS'] + " | " + criteria_dict['address'] + " into database.")
             except Exception as e:
                 print("Could not insert " + criteria_dict['address'] + " into database.  Database connection error.")
                 logging.warning("Could not insert " + criteria_dict['address'] + "into database.  Database connection "
@@ -381,27 +397,33 @@ class MLSDATA:
                 logging.warning(str(e))
         else:
             print("Database is not connected or cursor not filled.  Use function 'connectdb()' to establish")
+            print(str(self.cursor))
 
-    def connectdb(self, host='192.168.100.26', user='landsearchuser', password='1234', database='landsearch'):
+    def connect_db(self, host='192.168.100.26', user='landsearchuser', password='1234', database='landsearch'):
         """Connects to database and returns a cursor object"""
         self.cnx = mysql.connector.connect(host=host, user=user, password=password, database=database, buffered=True)
         self.cursor = self.cnx.cursor()
         return self.cursor
 
-    def closedb(self):
+    def close_db(self):
         """Cleanly close the db."""
         self.cursor.close()
         self.cnx.close()
 
-    def dbinsert(self, properties: list):
+    def db_insert(self, properties: list):
         """Inserts records into database.  Takes list of Property class objects"""
         if not properties == None:
             if not isinstance(properties, list):
                 raise TypeError('type list required')
             for property in properties:
-                if not self.checkdb({'MLS': property.MLS, 'address': property.address}):
-                    if self.live_google: self.getGoogle(
+                if not self.check_db({'MLS': property.MLS, 'address': property.address}):
+                    if self.live_google: self.get_google(
                         property)  ## <- This will populate distance and time fields if set TRUE
+                    else:
+                        print("NOT fetching google data.  Suppressed by settings in landsearch.conf")
+                        logging.warning("NOT fetching google data for " + property.address + ".  Suppressed by "
+                                                                                             "settings in "
+                                                                                             "landsearch.conf")
                     self.insertrecord(property)
                     self.new_listings.append(property)
                 else:
@@ -414,9 +436,6 @@ class MLSDATA:
         logging.info("Database Update Complete.")
         logging.info(str(len(self.new_listings)) + " new listings found.")
 
-    def alerts(self):
-        pass
-
     def email_results(self):
         global mymail
         sendto = ['stagl.mike@gmail.com', 'M_Stagl@hotmail.com']
@@ -424,10 +443,9 @@ class MLSDATA:
             ''' Send some kind of email! '''
             # If there are new listings, populate email ##
             if len(self.new_listings) > 0:
-                logging.debug("email_results" + str(self.email))
                 body = ''
                 data = []
-                subj = "New Real Estate Listings for " + str(datetime.date.today())
+                subj = str(len(self.new_listings)) + " New Real Estate Listings for " + str(datetime.date.today())
                 for listing in self.new_listings:
                     row = []
                     body += listing.MLS + " | " + listing.address + " | " + listing.acres + " | " + listing.price + " | " + listing.link + "\n"
@@ -464,6 +482,7 @@ class MLSDATA:
             print("Suppressing email based on landsearch.conf preferences.")
             logging.warning("Suppressing email based on landsearch.conf preferences.")
 
+
 if __name__ == '__main__':
 
     gmls = MLSDATA('GMLS')  # Create MLSDATA object
@@ -487,10 +506,8 @@ if __name__ == '__main__':
             for listing in mydata:
                 myresults.append(listing)
 
-    #  print(len(myresults))
-    #  print(myresults[0].address)
-    gmls.connectdb()
-    gmls.dbinsert(myresults)
-    gmls.closedb()
+    #gmls.connectdb()
+    gmls.db_insert(myresults)
+    #gmls.closedb()
 
     gmls.email_results()

landsearch.conf

@@ -1,6 +1,6 @@
 [Search]
 ## Accepts Gwinnett, Ballow, Hall, Jackson, and/or Walton.  Seperate multiple counties with comma.  Default is all 5 counties
-county = Barrow, Hall
+county = Barrow
 
 ## Accepts farm, house, and/or land.  Seperate multiples with comman.  Default is all 3
 ## land typically refers to land-lots
@@ -16,9 +16,13 @@ type = farm
 ; upper_bedrooms = 10
 
 ## It true, sends email of results (whether or not new listings are found)
-email = false
+email = true
+
+## Live_google will fetch actual time-to-school and time-to-work results from Google API.
+## There may be a cost for this service, thus it can be turned off for testing.
+live_google = True
 
 [Logging]
 log_file = landsearch.log
 ## log level accepts 10, 20, 30, 40, 50.  10 - Debug (most verbose) | 50 - Warning (least verbose)
-logging_level = 20
+logging_level = 10