|
|
@@ -2,81 +2,301 @@
|
|
|
|
|
|
from youtube_search import YoutubeSearch
|
|
|
import os
|
|
|
+import configparser
|
|
|
+import sys, getopt
|
|
|
+import requests, json
|
|
|
+import random, string
|
|
|
|
|
|
##Import list of songs, artists from list.txt
|
|
|
##parse this file into a list of dictionaries
|
|
|
##for each dictionary pair, search youtube and output formatted links in file
|
|
|
##
|
|
|
-##FUTURE - remove successful downloads from text file!!
|
|
|
-## -assign each line read a number
|
|
|
-## -remember numbers that are successful downloads
|
|
|
-## -at end of script, delete lines from those numbers
|
|
|
-## -OR-
|
|
|
-## -Find lines that match successful downloads
|
|
|
-
|
|
|
-DOWNLOAD=True
|
|
|
-
|
|
|
-musicfile="list.txt"
|
|
|
-
|
|
|
-music=[]
|
|
|
-songnum = 0
|
|
|
-with open(musicfile) as f:
|
|
|
- for line in f:
|
|
|
- song={}
|
|
|
- (key, val) = line.split(", ")
|
|
|
- songnum += 1
|
|
|
- song['songnum'] = songnum
|
|
|
- song['Title'] = key
|
|
|
- song['Artist'] = val.rstrip()
|
|
|
- song['raw'] = line
|
|
|
- music.append(song)
|
|
|
-f.close()
|
|
|
-
|
|
|
-logresults=[]
|
|
|
-linkresults=[]
|
|
|
-completed=[]
|
|
|
-for song in music:
|
|
|
- searchterm = song['Title'] + " " + song['Artist'] + ' lyrics'
|
|
|
- dictlink={}
|
|
|
- try:
|
|
|
- ytresult = YoutubeSearch(searchterm, max_results=1).to_dict()
|
|
|
- link = 'https://youtube.com' + ytresult[0]['link']
|
|
|
- dictlink['Title'] = song['Title']
|
|
|
- dictlink['Artist'] = song['Artist']
|
|
|
- dictlink['link'] = link
|
|
|
- linkresults.append(dictlink)
|
|
|
- logresults.append(song['Title'] + ", " + song['Artist'] + " Link Created")
|
|
|
- if DOWNLOAD:
|
|
|
- print("Attempting to download", song['Title'])
|
|
|
+## - Download over tor
|
|
|
+## - Check if Tor installed
|
|
|
+## - Warn user if NOT using tor
|
|
|
+##
|
|
|
+## - Multi-thread conversion to audio format
|
|
|
+##
|
|
|
+## - Logging and Verbosity levels
|
|
|
+##
|
|
|
+## - Check for dependencies
|
|
|
+## - Tor?
|
|
|
+## - Write Permissions in log files?
|
|
|
+## - YoutubeSearch
|
|
|
+##
|
|
|
+## - Differentiate between releases and masters when calling api
|
|
|
+##
|
|
|
+## - check for dependencies on start
|
|
|
+## - youtube-dl
|
|
|
+## - tor (if used)
|
|
|
+
|
|
|
+##Vars
|
|
|
+if os.path.exists('config.ini'):
|
|
|
+ config = configparser.ConfigParser()
|
|
|
+ config.read('config.ini')
|
|
|
+else:
|
|
|
+ print("Config.ini file not found in", os.getcwd())
|
|
|
+ print("Exiting...")
|
|
|
+ exit(1)
|
|
|
+
|
|
|
+VERSION="0.0.2"
|
|
|
+DOWNLOAD=config['DEFAULT'].getboolean('Download') #Download True/False
|
|
|
+MUSICFILE=config['DEFAULT']['Musicfile'] #location of text file containing songs
|
|
|
+RETRIES=config['DEFAULT'].getint('Retries') #Number of retries to search for songs
|
|
|
+ITERATOR=0 #Number or current tries
|
|
|
+VERBOSITY=config['DEFAULT'].getint('Verbosity')
|
|
|
+KEY=config['DEFAULT']['Key']
|
|
|
+DISCOG=""
|
|
|
+DESTFOLDER=""
|
|
|
+ALBUM=""
|
|
|
+ARTIST=""
|
|
|
+TESTFOLDER=config['DEFAULT'].getboolean('TestFolder')
|
|
|
+HELP= "Takes list.txt or Discogs.com Master/Release number \n" \
|
|
|
+ "And downloads albums by stripping the audio from Yuotube videos. \n" \
|
|
|
+ "USAGE: ytsearch [-flag] [Discog Num] \n" \
|
|
|
+ " -h This help file \n" \
|
|
|
+ " -d --discog set Discog.com Release or Master number \n" \
|
|
|
+ " -D --download override config.ini and set Download=True \n" \
|
|
|
+ " -f --file Allows quick download of a single Youtube link"
|
|
|
+JSONDATA=[]
|
|
|
+
|
|
|
+music=[] # list to hold dictionaries of songnum, Title, Artist
|
|
|
+logresults=[] # list to hold results of link creation attempts, and download attempts
|
|
|
+linkresults=[] #
|
|
|
+completed=[] # list to hold song numbers of completed downloads so they can be removed from MUSICFILE
|
|
|
+
|
|
|
+
|
|
|
+def msg(message, level):
|
|
|
+ ##Takes message and log level as arguments
|
|
|
+ ##Based on verbosity from config.ini, return messages to user
|
|
|
+ ## 1-ERROR, 2-WARN, 3-INFO, -1 [No flag, always show]
|
|
|
+ tlevel = {-1: '', 1: "ERROR", 2: "WARN", 3: "INFO"}
|
|
|
+ if level <= VERBOSITY:
|
|
|
+ print(tlevel.get(level), message)
|
|
|
+
|
|
|
+def arguments(argv):
|
|
|
+ msg("Starting arguments", 3)
|
|
|
+ try:
|
|
|
+ opts, args = getopt.getopt(argv, "hvDf:d:", ["discog", "help", "download", "version", "file"])
|
|
|
+ for opt, arg in opts:
|
|
|
+ if opt in ('-h', '--help'):
|
|
|
+ print(HELP)
|
|
|
+ sys.exit()
|
|
|
+ elif opt in ("-d", "--discog"):
|
|
|
+ global DISCOG
|
|
|
+ DISCOG = arg
|
|
|
+ msg("Discog number:" + DISCOG, 3)
|
|
|
+ elif opt in ("-D", "--download"):
|
|
|
+ global DOWNLOAD
|
|
|
+ DOWNLOAD = arg
|
|
|
+ msg("Override DOWNLOAD from agrs", 2)
|
|
|
+ elif opt in ("-v", "--version"):
|
|
|
+ msg("Version: " + VERSION, -1)
|
|
|
+ sys.exit()
|
|
|
+ elif opt in ("-f", "--file"):
|
|
|
+ msg("call singlesong with: " + arg, 3)
|
|
|
+ singlesong(arg)
|
|
|
+ except getopt.GetoptError as err:
|
|
|
+ msg("cannot get arguments, {0}".format(err), 1)
|
|
|
+
|
|
|
+def fetchjson(discogno, master=True, show=True):
|
|
|
+ msg("Starting fetchjson", 3)
|
|
|
+ if master:
|
|
|
+ url = 'https://api.discogs.com/masters/'
|
|
|
+ else:
|
|
|
+ url = 'https://api.discogs.com/releases/'
|
|
|
+ url = url + discogno
|
|
|
+ msg("Downloading " + url, 1)
|
|
|
+ r = requests.get(url)
|
|
|
+ global JSONDATA
|
|
|
+# JSONDATA = r.json()
|
|
|
+ JSONDATA = json.loads(r.text)
|
|
|
+ msg("fetchjson complete!", 3)
|
|
|
+ return JSONDATA
|
|
|
+
|
|
|
+def buildlist(jsondata, write=False):
|
|
|
+ ## takes raw jsons data from Discogs.com and extracts Album, Artist, and tracklist list
|
|
|
+ ## passes tracklist to gettracks to create list of track names
|
|
|
+ ## appends "," + artist to end of track names
|
|
|
+ ## calls buildfolders to create a home for the new file
|
|
|
+ ## writes list to list.txt in appropriate folder
|
|
|
+ msg("Staring Buildlist", 3)
|
|
|
+ try:
|
|
|
+ Artist = jsondata['artists'][0]['name']
|
|
|
+ Album = jsondata['title']
|
|
|
+ except:
|
|
|
+ msg("Could not read Artist or Album Name from Jsonfile", 1)
|
|
|
+ print(sys.exc_info()[0])
|
|
|
+ sys.exit()
|
|
|
+ if Artist.find( '(' ) != -1: ## Discovered a Artist 'Tool (2)' (Discogs 1181). This removes ()
|
|
|
+ Artist = Artist[:Artist.find( '(' )-1]
|
|
|
+ msg("Correcting Artist name to " + Artist, 2)
|
|
|
+
|
|
|
+ global ALBUM
|
|
|
+ try:
|
|
|
+ ALBUM = Album
|
|
|
+ msg("Set ALBUM var to: " + ALBUM, 3)
|
|
|
+ except Exception as e:
|
|
|
+ msg("Could not set ALBUM var." + e, 2)
|
|
|
+
|
|
|
+ tracks = gettracks(jsondata['tracklist'])
|
|
|
+
|
|
|
+ for i in range(len(tracks)):
|
|
|
+ tracks[i] = tracks[i] + ", " + Artist
|
|
|
+
|
|
|
+ print(tracks)
|
|
|
+
|
|
|
+ if write:
|
|
|
+ with open('list.txt', 'w') as f:
|
|
|
+ for j in range (len(tracks)):
|
|
|
+ f.write(tracks[j] + "\n")
|
|
|
+ f.close()
|
|
|
+
|
|
|
+def gettracks(tracks):
|
|
|
+ ## takes raw json data from Discogs.com and creates a tracklist for the album
|
|
|
+ ## This will return a list
|
|
|
+ msg("Starting gettracks", 3)
|
|
|
+ goodtracks = []
|
|
|
+
|
|
|
+ for track in tracks:
|
|
|
+ goodtracks.append(track['title'].replace(',',''))
|
|
|
+
|
|
|
+ return goodtracks
|
|
|
+
|
|
|
+def randomizer(length=8):
|
|
|
+ #Creates a Random directory name
|
|
|
+ randoms = string.ascii_letters + string.digits
|
|
|
+ return ''.join((random.choice(randoms) for i in range(length)))
|
|
|
+
|
|
|
+def buildfolders(artist, album=""):
|
|
|
+ ## Takes raw json data and creates foldes in parent_directory for Artist/Album
|
|
|
+ msg("buildfolders started", 3)
|
|
|
+ if len(ALBUM) == 0:
|
|
|
+ album="UNKNOWN ALBUM"
|
|
|
+ else:
|
|
|
+ album = ALBUM
|
|
|
+# msg("buildfolders local album is set to: " + album, 3)
|
|
|
+ global DESTFOLDER
|
|
|
+# DESTFOLDER = randomizer() + "/" + artist + "/" + album
|
|
|
+ DESTFOLDER = artist + "/" + album + "/"
|
|
|
+ if TESTFOLDER:
|
|
|
+ DESTFOLDER = randomizer() + "/" + DESTFOLDER
|
|
|
+ try:
|
|
|
+ os.makedirs(DESTFOLDER)
|
|
|
+ msg("Folder " + DESTFOLDER + " created", 2)
|
|
|
+ except:
|
|
|
+ msg("Could not create destination folder!", 1)
|
|
|
+
|
|
|
+def readlist(file):
|
|
|
+ msg("Starting readlist", 3)
|
|
|
+ if not os.path.exists(MUSICFILE):
|
|
|
+ msg("List.txt file not found. Exiting", 1)
|
|
|
+ sys.exit()
|
|
|
+ ##Open list.txt, read into music[]
|
|
|
+ songnum = 0
|
|
|
+ with open(file) as f:
|
|
|
+ for line in f:
|
|
|
+ song={}
|
|
|
+ (key, val) = line.split(", ")
|
|
|
+ songnum += 1
|
|
|
+ song['songnum'] = songnum
|
|
|
+ song['Title'] = key
|
|
|
+ song['Artist'] = val.rstrip()
|
|
|
+ music.append(song)
|
|
|
+ f.close()
|
|
|
+
|
|
|
+ return music
|
|
|
+
|
|
|
+def parselist(musiclist):
|
|
|
+ msg("Starting parselist", 3)
|
|
|
+ global ITERATOR
|
|
|
+ if ITERATOR == 0 and DOWNLOAD:
|
|
|
+ buildfolders(musiclist[0]['Artist'])
|
|
|
+ ## Build File Structure using buildfolders
|
|
|
+ ## Somehow have downloads save to this folder without knowing the file name?
|
|
|
+ ## Perhaps with a flag in youtube-dl
|
|
|
+ ITERATOR+=1
|
|
|
+ print(musiclist)
|
|
|
+ for song in musiclist:
|
|
|
+ searchterm = song['Title'] + " " + song['Artist'] + ' lyrics HD'
|
|
|
+# searchterm = song['Title'] + " " + song['Artist']
|
|
|
+ dictlink={}
|
|
|
+ try:
|
|
|
+ ytresult = YoutubeSearch(searchterm, max_results=1).to_dict() ##increase timeout!!
|
|
|
+ link = 'https://youtube.com' + ytresult[0]['link']
|
|
|
+ logresults.append(song['Title'] + ", " + song['Artist'] + " Link Created")
|
|
|
+ if DOWNLOAD:
|
|
|
+ msg("Attempting to download " + song['Title'], 2)
|
|
|
+ downloadsong(link, song)
|
|
|
+ else:
|
|
|
+ print("Not downloading " + song['Title'] + ". Change this in config.ini")
|
|
|
+ except Exception as ex:
|
|
|
+ print(ex)
|
|
|
+ logresults.append(song['Title'] + ", " + song['Artist'] + " COULD NOT CREATE LINK")
|
|
|
+ if DOWNLOAD:
|
|
|
+ cleanup(MUSICFILE)
|
|
|
+
|
|
|
+def downloadsong(link, song, path=""):
|
|
|
+ msg("Starting Downloadsong", 3)
|
|
|
+ if len(DESTFOLDER) > 1:
|
|
|
+ path=DESTFOLDER
|
|
|
+ msg("DESTFOLDER var = " + DESTFOLDER, 3)
|
|
|
+ msg("local path var = " + path, 3)
|
|
|
try:
|
|
|
- os.system("youtube-dl --extract-audio --audio-format mp3 --output '%(title)s.%(ext)s' --ignore-errors " + link)
|
|
|
+ os.system("youtube-dl --extract-audio --audio-format best --audio-quality 0 --output '''" + path + "%(title)s.%(ext)s' --ignore-errors " + link)
|
|
|
completed.append(song['songnum'])
|
|
|
logresults.append(song['Title'] + ", " + song['Artist'] + " Audio downloaded")
|
|
|
- print("Download Complete!")
|
|
|
+ msg(song['Title'] + " Download Complete!", 2)
|
|
|
except e as youtubedlexception:
|
|
|
logresults.append(song['Title'] + ", " + song['Artist'] + " FAILED TO DOWNLOAD SONG (youtube-dl)")
|
|
|
print(youtubedlexception)
|
|
|
- else:
|
|
|
- print("ERROR: Not Downloading for some reason")
|
|
|
- except:
|
|
|
- logresults.append(song['Title'] + ", " + song['Artist'] + " COULD NOT CREATE LINK")
|
|
|
|
|
|
-print("------------")
|
|
|
-for r in logresults:
|
|
|
- print(r)
|
|
|
+def singlesong(link):
|
|
|
+ try:
|
|
|
+ os.system("youtube-dl --extract-audio --audio-format best --audio-quality 0 --output '%(title)s.%(ext)s' --ignore-errors " + link)
|
|
|
+ except Exception as e:
|
|
|
+ msg("Could not download file. " + e, 1)
|
|
|
+ sys.exit()
|
|
|
+
|
|
|
+def cleanup(file):
|
|
|
+ print("Cleaning completed files from list")
|
|
|
+ print("Completed Downloads:", completed)
|
|
|
+
|
|
|
+ linenum=0
|
|
|
+ count=0
|
|
|
+ with open(file, "r") as f:
|
|
|
+ lines = f.readlines()
|
|
|
+ with open(file, "w") as f:
|
|
|
+ for line in lines:
|
|
|
+ linenum += 1
|
|
|
+ if linenum not in completed:
|
|
|
+ f.write(line)
|
|
|
+ count += 1
|
|
|
+
|
|
|
+ f.close()
|
|
|
+
|
|
|
+ if count >=1:
|
|
|
+ print(count, "TRACKS REMAIN")
|
|
|
+ print(RETRIES - ITERATOR, "tries remaining")
|
|
|
+ if ITERATOR <= RETRIES:
|
|
|
+ print("Retrying")
|
|
|
+ global music
|
|
|
+ music = []
|
|
|
+ parselist(readlist(MUSICFILE))
|
|
|
+ else:
|
|
|
+ msg("All downloads complete!", -1)
|
|
|
|
|
|
+if __name__ == "__main__":
|
|
|
|
|
|
-print(completed)
|
|
|
+# msg("Starting ytsearch...", -1)
|
|
|
|
|
|
-print("Cleaning completed files from list")
|
|
|
+ arguments(sys.argv[1:])
|
|
|
|
|
|
-linenum=0
|
|
|
-with open(musicfile, "r") as f:
|
|
|
- lines = f.readlines()
|
|
|
-with open(musicfile, "w") as f:
|
|
|
- for line in lines:
|
|
|
- linenum += 1
|
|
|
- if linenum not in completed:
|
|
|
- f.write(line)
|
|
|
+ if DISCOG != "":
|
|
|
+ msg("DISCOG found, fetch json", 3)
|
|
|
+# print(fetchjson(DISCOG))
|
|
|
+ buildlist(fetchjson(DISCOG), True)
|
|
|
+ readlist(MUSICFILE)
|
|
|
+ parselist(music)
|
|
|
|
|
|
-f.close()
|
|
|
+ print("ytsearch complete, exiting")
|
mike