# -*- coding: UTF-8 -*-
"""
    OpenScrapers Module

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

# Addon Name: OpenScrapers Module
# Addon id: script.module.openscrapers

import cookielib
import os
import re
import sys
import traceback
import urllib
import urllib2

import xbmc
import xbmcaddon

profile = functions_dir = xbmc.translatePath(xbmcaddon.Addon().getAddonInfo('profile').decode('utf-8'))

try:
    from sqlite3 import dbapi2 as database
except:
    from pysqlite2 import dbapi2 as database

from openscrapers.modules import client
from openscrapers.modules import control


def fetch(regex):
    try:
        cacheFile = os.path.join(control.dataPath, 'regex.db')
        dbcon = database.connect(cacheFile)
        dbcur = dbcon.cursor()
        dbcur.execute("SELECT * FROM regex WHERE regex = '%s'" % regex)
        regex = dbcur.fetchone()[1]
        return regex
    except:
        return


def insert(data):
    try:
        control.makeFile(control.dataPath)
        cacheFile = os.path.join(control.dataPath, 'regex.db')
        dbcon = database.connect(cacheFile)
        dbcur = dbcon.cursor()
        dbcur.execute("CREATE TABLE IF NOT EXISTS regex (""regex TEXT, ""response TEXT, ""UNIQUE(regex)"");")
        for i in data:
            try:
                dbcur.execute("INSERT INTO regex Values (?, ?)", (i['regex'], i['response']))
            except:
                pass
        dbcon.commit()
    except:
        return


def clear():
    try:
        cacheFile = os.path.join(control.dataPath, 'regex.db')
        dbcon = database.connect(cacheFile)
        dbcur = dbcon.cursor()
        dbcur.execute("DROP TABLE IF EXISTS regex")
        dbcur.execute("VACUUM")
        dbcon.commit()
    except:
        pass


def resolve(regex):
    try:
        vanilla = re.compile('(<regex>.+)', re.MULTILINE | re.DOTALL).findall(regex)[0]
        cddata = re.compile('<\!\[CDATA\[(.+?)\]\]>', re.MULTILINE | re.DOTALL).findall(regex)
        for i in cddata:
            regex = regex.replace('<![CDATA[' + i + ']]>', urllib.quote_plus(i))

        regexs = re.compile('(<regex>.+)', re.MULTILINE | re.DOTALL).findall(regex)[0]
        regexs = re.compile('<regex>(.+?)</regex>', re.MULTILINE | re.DOTALL).findall(regexs)
        regexs = [re.compile('<(.+?)>(.*?)</.+?>', re.MULTILINE | re.DOTALL).findall(i) for i in regexs]

        regexs = [dict([(client.replaceHTMLCodes(x[0]), client.replaceHTMLCodes(urllib.unquote_plus(x[1]))) for x in i])
                  for i in regexs]
        regexs = [(i['name'], i) for i in regexs]
        regexs = dict(regexs)

        url = regex.split('<regex>', 1)[0].strip()
        url = client.replaceHTMLCodes(url)
        url = url.encode('utf-8')

        r = getRegexParsed(regexs, url)

        try:
            ln = ''
            ret = r[1]
            listrepeat = r[2]['listrepeat']
            regexname = r[2]['name']

            for obj in ret:
                try:
                    item = listrepeat
                    for i in range(len(obj) + 1):
                        item = item.replace('[%s.param%s]' % (regexname, str(i)), obj[i - 1])

                    item2 = vanilla
                    for i in range(len(obj) + 1):
                        item2 = item2.replace('[%s.param%s]' % (regexname, str(i)), obj[i - 1])

                    item2 = re.compile('(<regex>.+?</regex>)', re.MULTILINE | re.DOTALL).findall(item2)
                    item2 = [x for x in item2 if not '<name>%s</name>' % regexname in x]
                    item2 = ''.join(item2)

                    ln += '\n<item>%s\n%s</item>\n' % (item, item2)
                except:
                    pass

            return ln
        except:
            pass

        if r[1] == True:
            return r[0]
    except:
        return


class NoRedirection(urllib2.HTTPErrorProcessor):
    def http_response(self, request, response):
        return response

    https_response = http_response


def getRegexParsed(regexs, url, cookieJar=None, forCookieJarOnly=False, recursiveCall=False, cachedPages={},
                   rawPost=False, cookie_jar_file=None):  # 0,1,2 = URL, regexOnly, CookieJarOnly
    # cachedPages = {}
    # print 'url',url
    doRegexs = re.compile('\$doregex\[([^\]]*)\]').findall(url)
    #        print 'doRegexs',doRegexs,regexs
    setresolved = True
    for k in doRegexs:
        if k in regexs:
            # print 'processing ' ,k
            m = regexs[k]
            # print m
            cookieJarParam = False
            if 'cookiejar' in m:  # so either create or reuse existing jar
                # print 'cookiejar exists',m['cookiejar']
                cookieJarParam = m['cookiejar']
                if '$doregex' in cookieJarParam:
                    cookieJar = getRegexParsed(regexs, m['cookiejar'], cookieJar, True, True, cachedPages)
                    cookieJarParam = True
                else:
                    cookieJarParam = True
            # print 'm[cookiejar]',m['cookiejar'],cookieJar
            if cookieJarParam:
                if cookieJar == None:
                    # print 'create cookie jar'
                    cookie_jar_file = None
                    if 'open[' in m['cookiejar']:
                        cookie_jar_file = m['cookiejar'].split('open[')[1].split(']')[0]
                    #                            print 'cookieJar from file name',cookie_jar_file

                    cookieJar = getCookieJar(cookie_jar_file)
                    #                        print 'cookieJar from file',cookieJar
                    if cookie_jar_file:
                        saveCookieJar(cookieJar, cookie_jar_file)
                    # import cookielib
                    # cookieJar = cookielib.LWPCookieJar()
                    # print 'cookieJar new',cookieJar
                elif 'save[' in m['cookiejar']:
                    cookie_jar_file = m['cookiejar'].split('save[')[1].split(']')[0]
                    complete_path = os.path.join(profile, cookie_jar_file)
                    #                        print 'complete_path',complete_path
                    saveCookieJar(cookieJar, cookie_jar_file)

            if m['page'] and '$doregex' in m['page']:
                pg = getRegexParsed(regexs, m['page'], cookieJar, recursiveCall=True, cachedPages=cachedPages)
                if len(pg) == 0:
                    pg = 'http://regexfailed'
                m['page'] = pg

            if 'setcookie' in m and m['setcookie'] and '$doregex' in m['setcookie']:
                m['setcookie'] = getRegexParsed(regexs, m['setcookie'], cookieJar, recursiveCall=True,
                                                cachedPages=cachedPages)
            if 'appendcookie' in m and m['appendcookie'] and '$doregex' in m['appendcookie']:
                m['appendcookie'] = getRegexParsed(regexs, m['appendcookie'], cookieJar, recursiveCall=True,
                                                   cachedPages=cachedPages)

            if 'post' in m and '$doregex' in m['post']:
                m['post'] = getRegexParsed(regexs, m['post'], cookieJar, recursiveCall=True, cachedPages=cachedPages)
            #                    print 'post is now',m['post']

            if 'rawpost' in m and '$doregex' in m['rawpost']:
                m['rawpost'] = getRegexParsed(regexs, m['rawpost'], cookieJar, recursiveCall=True,
                                              cachedPages=cachedPages, rawPost=True)
                # print 'rawpost is now',m['rawpost']

            if 'rawpost' in m and '$epoctime$' in m['rawpost']:
                m['rawpost'] = m['rawpost'].replace('$epoctime$', getEpocTime())

            if 'rawpost' in m and '$epoctime2$' in m['rawpost']:
                m['rawpost'] = m['rawpost'].replace('$epoctime2$', getEpocTime2())

            link = ''
            if m['page'] and m['page'] in cachedPages and not 'ignorecache' in m and forCookieJarOnly == False:
                # print 'using cache page',m['page']
                link = cachedPages[m['page']]
            else:
                if m['page'] and not m['page'] == '' and m['page'].startswith('http'):
                    if '$epoctime$' in m['page']:
                        m['page'] = m['page'].replace('$epoctime$', getEpocTime())
                    if '$epoctime2$' in m['page']:
                        m['page'] = m['page'].replace('$epoctime2$', getEpocTime2())

                    # print 'Ingoring Cache',m['page']
                    page_split = m['page'].split('|')
                    pageUrl = page_split[0]
                    header_in_page = None
                    if len(page_split) > 1:
                        header_in_page = page_split[1]

                    #                            if
                    #                            proxy = urllib2.ProxyHandler({ ('https' ? proxytouse[:5]=="https":"http") : proxytouse})
                    #                            opener = urllib2.build_opener(proxy)
                    #                            urllib2.install_opener(opener)

                    #                        import urllib2
                    #                        print 'urllib2.getproxies',urllib2.getproxies()
                    current_proxies = urllib2.ProxyHandler(urllib2.getproxies())

                    # print 'getting pageUrl',pageUrl
                    req = urllib2.Request(pageUrl)
                    if 'proxy' in m:
                        proxytouse = m['proxy']
                        #                            print 'proxytouse',proxytouse
                        #                            urllib2.getproxies= lambda: {}
                        if pageUrl[:5] == "https":
                            proxy = urllib2.ProxyHandler({'https': proxytouse})
                            # req.set_proxy(proxytouse, 'https')
                        else:
                            proxy = urllib2.ProxyHandler({'http': proxytouse})
                            # req.set_proxy(proxytouse, 'http')
                        opener = urllib2.build_opener(proxy)
                        urllib2.install_opener(opener)

                    req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/14.0.1')
                    proxytouse = None

                    if 'referer' in m:
                        req.add_header('Referer', m['referer'])
                    if 'accept' in m:
                        req.add_header('Accept', m['accept'])
                    if 'agent' in m:
                        req.add_header('User-agent', m['agent'])
                    if 'x-req' in m:
                        req.add_header('X-Requested-With', m['x-req'])
                    if 'x-addr' in m:
                        req.add_header('x-addr', m['x-addr'])
                    if 'x-forward' in m:
                        req.add_header('X-Forwarded-For', m['x-forward'])
                    if 'setcookie' in m:
                        #                            print 'adding cookie',m['setcookie']
                        req.add_header('Cookie', m['setcookie'])
                    if 'appendcookie' in m:
                        #                            print 'appending cookie to cookiejar',m['appendcookie']
                        cookiestoApend = m['appendcookie']
                        cookiestoApend = cookiestoApend.split(';')
                        for h in cookiestoApend:
                            n, v = h.split('=')
                            w, n = n.split(':')
                            ck = cookielib.Cookie(version=0, name=n, value=v, port=None, port_specified=False, domain=w,
                                                  domain_specified=False, domain_initial_dot=False, path='/',
                                                  path_specified=True, secure=False, expires=None, discard=True,
                                                  comment=None, comment_url=None, rest={'HttpOnly': None},
                                                  rfc2109=False)
                            cookieJar.set_cookie(ck)
                    if 'origin' in m:
                        req.add_header('Origin', m['origin'])
                    if header_in_page:
                        header_in_page = header_in_page.split('&')
                        for h in header_in_page:
                            n, v = h.split('=')
                            req.add_header(n, v)

                    if not cookieJar == None:
                        #                            print 'cookieJarVal',cookieJar
                        cookie_handler = urllib2.HTTPCookieProcessor(cookieJar)
                        opener = urllib2.build_opener(cookie_handler, urllib2.HTTPBasicAuthHandler(),
                                                      urllib2.HTTPHandler())
                        opener = urllib2.install_opener(opener)
                        #                            print 'noredirect','noredirect' in m

                        if 'noredirect' in m:
                            opener = urllib2.build_opener(cookie_handler, NoRedirection, urllib2.HTTPBasicAuthHandler(),
                                                          urllib2.HTTPHandler())
                            opener = urllib2.install_opener(opener)
                    elif 'noredirect' in m:
                        opener = urllib2.build_opener(NoRedirection, urllib2.HTTPBasicAuthHandler(),
                                                      urllib2.HTTPHandler())
                        opener = urllib2.install_opener(opener)

                    if 'connection' in m:
                        #                            print '..........................connection//////.',m['connection']
                        from keepalive import HTTPHandler
                        keepalive_handler = HTTPHandler()
                        opener = urllib2.build_opener(keepalive_handler)
                        urllib2.install_opener(opener)

                    # print 'after cookie jar'
                    post = None

                    if 'post' in m:
                        postData = m['post']
                        # if '$LiveStreamRecaptcha' in postData:
                        #    (captcha_challenge,catpcha_word,idfield)=processRecaptcha(m['page'],cookieJar)
                        #    if captcha_challenge:
                        #        postData=postData.replace('$LiveStreamRecaptcha','manual_recaptcha_challenge_field:'+captcha_challenge+',recaptcha_response_field:'+catpcha_word+',id:'+idfield)
                        splitpost = postData.split(',');
                        post = {}
                        for p in splitpost:
                            n = p.split(':')[0];
                            v = p.split(':')[1];
                            post[n] = v
                        post = urllib.urlencode(post)

                    if 'rawpost' in m:
                        post = m['rawpost']
                        # if '$LiveStreamRecaptcha' in post:
                        #    (captcha_challenge,catpcha_word,idfield)=processRecaptcha(m['page'],cookieJar)
                        #    if captcha_challenge:
                        #       post=post.replace('$LiveStreamRecaptcha','&manual_recaptcha_challenge_field='+captcha_challenge+'&recaptcha_response_field='+catpcha_word+'&id='+idfield)
                    link = ''
                    try:

                        if post:
                            response = urllib2.urlopen(req, post)
                        else:
                            response = urllib2.urlopen(req)
                        if response.info().get('Content-Encoding') == 'gzip':
                            from StringIO import StringIO
                            import gzip
                            buf = StringIO(response.read())
                            f = gzip.GzipFile(fileobj=buf)
                            link = f.read()
                        else:
                            link = response.read()

                        if 'proxy' in m and not current_proxies is None:
                            urllib2.install_opener(urllib2.build_opener(current_proxies))

                        link = javascriptUnEscape(link)
                        # print repr(link)
                        # print link This just print whole webpage in LOG
                        if 'includeheaders' in m:
                            # link+=str(response.headers.get('Set-Cookie'))
                            link += '$$HEADERS_START$$:'
                            for b in response.headers:
                                link += b + ':' + response.headers.get(b) + '\n'
                            link += '$$HEADERS_END$$:'
                        #                        print link

                        response.close()
                    except:
                        pass
                    cachedPages[m['page']] = link
                    # print link
                    # print 'store link for',m['page'],forCookieJarOnly

                    if forCookieJarOnly:
                        return cookieJar  # do nothing
                elif m['page'] and not m['page'].startswith('http'):
                    if m['page'].startswith('$pyFunction:'):
                        val = doEval(m['page'].split('$pyFunction:')[1], '', cookieJar, m)
                        if forCookieJarOnly:
                            return cookieJar  # do nothing
                        link = val
                        link = javascriptUnEscape(link)
                    else:
                        link = m['page']

            if '$doregex' in m['expres']:
                m['expres'] = getRegexParsed(regexs, m['expres'], cookieJar, recursiveCall=True,
                                             cachedPages=cachedPages)

            if not m['expres'] == '':
                # print 'doing it ',m['expres']
                if '$LiveStreamCaptcha' in m['expres']:
                    val = askCaptcha(m, link, cookieJar)
                    # print 'url and val',url,val
                    url = url.replace("$doregex[" + k + "]", val)

                elif m['expres'].startswith('$pyFunction:') or '#$pyFunction' in m['expres']:
                    # print 'expeeeeeeeeeeeeeeeeeee',m['expres']
                    val = ''
                    if m['expres'].startswith('$pyFunction:'):
                        val = doEval(m['expres'].split('$pyFunction:')[1], link, cookieJar, m)
                    else:
                        val = doEvalFunction(m['expres'], link, cookieJar, m)
                    if 'ActivateWindow' in m['expres']: return
                    if forCookieJarOnly:
                        return cookieJar  # do nothing
                    if 'listrepeat' in m:
                        listrepeat = m['listrepeat']
                        return listrepeat, eval(val), m, regexs, cookieJar

                    try:
                        url = url.replace(u"$doregex[" + k + "]", val)
                    except:
                        url = url.replace("$doregex[" + k + "]", val.decode("utf-8"))
                else:
                    if 'listrepeat' in m:
                        listrepeat = m['listrepeat']
                        ret = re.findall(m['expres'], link)
                        return listrepeat, ret, m, regexs

                    val = ''
                    if not link == '':
                        # print 'link',link
                        reg = re.compile(m['expres']).search(link)
                        try:
                            val = reg.group(1).strip()
                        except:
                            traceback.print_exc()
                    elif m['page'] == '' or m['page'] == None:
                        val = m['expres']

                    if rawPost:
                        #                            print 'rawpost'
                        val = urllib.quote_plus(val)
                    if 'htmlunescape' in m:
                        # val=urllib.unquote_plus(val)
                        import HTMLParser
                        val = HTMLParser.HTMLParser().unescape(val)
                    try:
                        url = url.replace("$doregex[" + k + "]", val)
                    except:
                        url = url.replace("$doregex[" + k + "]", val.decode("utf-8"))
                    # print 'ur',url
                    # return val
            else:
                url = url.replace("$doregex[" + k + "]", '')
    if '$epoctime$' in url:
        url = url.replace('$epoctime$', getEpocTime())
    if '$epoctime2$' in url:
        url = url.replace('$epoctime2$', getEpocTime2())

    if '$GUID$' in url:
        import uuid
        url = url.replace('$GUID$', str(uuid.uuid1()).upper())
    if '$get_cookies$' in url:
        url = url.replace('$get_cookies$', getCookiesString(cookieJar))

    if recursiveCall: return url
    # print 'final url',repr(url)
    if url == "":
        return
    else:
        return url, setresolved


def get_unwise(str_eval):
    page_value = ""
    try:
        ss = "w,i,s,e=(" + str_eval + ')'
        exec (ss)
        page_value = unwise_func(w, i, s, e)
    except:
        traceback.print_exc(file=sys.stdout)
    # print 'unpacked',page_value
    return page_value


def unwise_func(w, i, s, e):
    lIll = 0;
    ll1I = 0;
    Il1l = 0;
    ll1l = [];
    l1lI = [];
    while True:
        if (lIll < 5):
            l1lI.append(w[lIll])
        elif (lIll < len(w)):
            ll1l.append(w[lIll]);
        lIll += 1;
        if (ll1I < 5):
            l1lI.append(i[ll1I])
        elif (ll1I < len(i)):
            ll1l.append(i[ll1I])
        ll1I += 1;
        if (Il1l < 5):
            l1lI.append(s[Il1l])
        elif (Il1l < len(s)):
            ll1l.append(s[Il1l]);
        Il1l += 1;
        if (len(w) + len(i) + len(s) + len(e) == len(ll1l) + len(l1lI) + len(e)):
            break;

    lI1l = ''.join(ll1l)  # .join('');
    I1lI = ''.join(l1lI)  # .join('');
    ll1I = 0;
    l1ll = [];
    for lIll in range(0, len(ll1l), 2):
        # print 'array i',lIll,len(ll1l)
        ll11 = -1;
        if (ord(I1lI[ll1I]) % 2):
            ll11 = 1;
        # print 'val is ', lI1l[lIll: lIll+2]
        l1ll.append(chr(int(lI1l[lIll: lIll + 2], 36) - ll11));
        ll1I += 1;
        if (ll1I >= len(l1lI)):
            ll1I = 0;
    ret = ''.join(l1ll)
    if 'eval(function(w,i,s,e)' in ret:
        #        print 'STILL GOing'
        ret = re.compile('eval\(function\(w,i,s,e\).*}\((.*?)\)').findall(ret)[0]
        return get_unwise(ret)
    else:
        #        print 'FINISHED'
        return ret


def get_unpacked(page_value, regex_for_text='', iterations=1, total_iteration=1):
    try:
        reg_data = None
        if page_value.startswith("http"):
            page_value = getUrl(page_value)
        #        print 'page_value',page_value
        if regex_for_text and len(regex_for_text) > 0:
            try:
                page_value = re.compile(regex_for_text).findall(page_value)[0]  # get the js variable
            except:
                return 'NOTPACKED'

        page_value = unpack(page_value, iterations, total_iteration)
    except:
        page_value = 'UNPACKEDFAILED'
        traceback.print_exc(file=sys.stdout)
    #    print 'unpacked',page_value
    if 'sav1live.tv' in page_value:
        page_value = page_value.replace('sav1live.tv', 'sawlive.tv')  # quick fix some bug somewhere
    #        print 'sav1 unpacked',page_value
    return page_value


def unpack(sJavascript, iteration=1, totaliterations=2):
    #    print 'iteration',iteration
    if sJavascript.startswith('var _0xcb8a='):
        aSplit = sJavascript.split('var _0xcb8a=')
        ss = "myarray=" + aSplit[1].split("eval(")[0]
        exec (ss)
        a1 = 62
        c1 = int(aSplit[1].split(",62,")[1].split(',')[0])
        p1 = myarray[0]
        k1 = myarray[3]
        with open('temp file' + str(iteration) + '.js', "wb") as filewriter:
            filewriter.write(str(k1))
        # aa=1/0
    else:

        if "rn p}('" in sJavascript:
            aSplit = sJavascript.split("rn p}('")
        else:
            aSplit = sJavascript.split("rn A}('")
        #        print aSplit

        p1, a1, c1, k1 = ('', '0', '0', '')

        ss = "p1,a1,c1,k1=('" + aSplit[1].split(".spli")[0] + ')'
        exec (ss)
    k1 = k1.split('|')
    aSplit = aSplit[1].split("))'")

    e = ''
    d = ''  # 32823

    # sUnpacked = str(__unpack(p, a, c, k, e, d))
    sUnpacked1 = str(__unpack(p1, a1, c1, k1, e, d, iteration))

    # print sUnpacked[:200]+'....'+sUnpacked[-100:], len(sUnpacked)
    #    print sUnpacked1[:200]+'....'+sUnpacked1[-100:], len(sUnpacked1)

    # exec('sUnpacked1="'+sUnpacked1+'"')
    if iteration >= totaliterations:
        #        print 'final res',sUnpacked1[:200]+'....'+sUnpacked1[-100:], len(sUnpacked1)
        return sUnpacked1  # .replace('\\\\', '\\')
    else:
        #        print 'final res for this iteration is',iteration
        return unpack(sUnpacked1,
                      iteration + 1)  # .replace('\\', ''),iteration)#.replace('\\', '');#unpack(sUnpacked.replace('\\', ''))


def __unpack(p, a, c, k, e, d, iteration, v=1):
    # with open('before file'+str(iteration)+'.js', "wb") as filewriter:
    #    filewriter.write(str(p))
    while (c >= 1):
        c = c - 1
        if (k[c]):
            aa = str(__itoaNew(c, a))
            if v == 1:
                p = re.sub('\\b' + aa + '\\b', k[c], p)  # THIS IS Bloody slow!
            else:
                p = findAndReplaceWord(p, aa, k[c])

            # p=findAndReplaceWord(p,aa,k[c])

    # with open('after file'+str(iteration)+'.js', "wb") as filewriter:
    #    filewriter.write(str(p))
    return p


def __itoa(num, radix):
    #    print 'num red',num, radix
    result = ""
    if num == 0: return '0'
    while num > 0:
        result = "0123456789abcdefghijklmnopqrstuvwxyz"[num % radix] + result
        num /= radix
    return result


def __itoaNew(cc, a):
    aa = "" if cc < a else __itoaNew(int(cc / a), a)
    cc = (cc % a)
    bb = chr(cc + 29) if cc > 35 else str(__itoa(cc, 36))
    return aa + bb


def findAndReplaceWord(source_str, word_to_find, replace_with):
    splits = None
    splits = source_str.split(word_to_find)
    if len(splits) > 1:
        new_string = []
        current_index = 0
        for current_split in splits:
            # print 'here',i
            new_string.append(current_split)
            val = word_to_find  # by default assume it was wrong to split

            # if its first one and item is blank then check next item is valid or not
            if current_index == len(splits) - 1:
                val = ''  # last one nothing to append normally
            else:
                if len(current_split) == 0:  # if blank check next one with current split value
                    if (len(splits[current_index + 1]) == 0 and word_to_find[
                        0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_') or (
                            len(splits[current_index + 1]) > 0 and splits[current_index + 1][
                        0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_'):  # first just just check next
                        val = replace_with
                # not blank, then check current endvalue and next first value
                else:
                    if (splits[current_index][-1].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_') and ((len(
                            splits[current_index + 1]) == 0 and word_to_find[
                                                                                                                    0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_') or (
                                                                                                                       len(
                                                                                                                           splits[
                                                                                                                               current_index + 1]) > 0 and
                                                                                                                       splits[
                                                                                                                           current_index + 1][
                                                                                                                           0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_')):  # first just just check next
                        val = replace_with

            new_string.append(val)
            current_index += 1
        # aaaa=1/0
        source_str = ''.join(new_string)
    return source_str


def re_me(data, re_patten):
    match = ''
    m = re.search(re_patten, data)
    if m != None:
        match = m.group(1)
    else:
        match = ''
    return match


def getCookiesString(cookieJar):
    try:
        cookieString = ""
        for index, cookie in enumerate(cookieJar):
            cookieString += cookie.name + "=" + cookie.value + ";"
    except:
        pass
    # print 'cookieString',cookieString
    return cookieString


def saveCookieJar(cookieJar, COOKIEFILE):
    try:
        complete_path = os.path.join(profile, COOKIEFILE)
        cookieJar.save(complete_path, ignore_discard=True)
    except:
        pass


def getCookieJar(COOKIEFILE):
    cookieJar = None
    if COOKIEFILE:
        try:
            complete_path = os.path.join(profile, COOKIEFILE)
            cookieJar = cookielib.LWPCookieJar()
            cookieJar.load(complete_path, ignore_discard=True)
        except:
            cookieJar = None

    if not cookieJar:
        cookieJar = cookielib.LWPCookieJar()

    return cookieJar


def doEval(fun_call, page_data, Cookie_Jar, m):
    ret_val = ''
    # print fun_call
    if functions_dir not in sys.path:
        sys.path.append(functions_dir)

    #    print fun_call
    try:
        py_file = 'import ' + fun_call.split('.')[0]
        #        print py_file,sys.path
        exec (py_file)
    #        print 'done'
    except:
        # print 'error in import'
        traceback.print_exc(file=sys.stdout)
    #    print 'ret_val='+fun_call
    exec ('ret_val=' + fun_call)
    #    print ret_val
    # exec('ret_val=1+1')
    try:
        return str(ret_val)
    except:
        return ret_val


def doEvalFunction(fun_call, page_data, Cookie_Jar, m):
    #    print 'doEvalFunction'
    ret_val = ''
    if functions_dir not in sys.path:
        sys.path.append(functions_dir)

    f = open(functions_dir + "/LSProdynamicCode.py", "w")
    f.write(fun_call);
    f.close()
    import LSProdynamicCode
    ret_val = LSProdynamicCode.GetLSProData(page_data, Cookie_Jar, m)
    try:
        return str(ret_val)
    except:
        return ret_val


def getUrl(url, cookieJar=None, post=None, timeout=20, headers=None, noredir=False):
    cookie_handler = urllib2.HTTPCookieProcessor(cookieJar)

    if noredir:
        opener = urllib2.build_opener(NoRedirection, cookie_handler, urllib2.HTTPBasicAuthHandler(),
                                      urllib2.HTTPHandler())
    else:
        opener = urllib2.build_opener(cookie_handler, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
    # opener = urllib2.install_opener(opener)
    req = urllib2.Request(url)
    req.add_header('User-Agent',
                   'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36')
    if headers:
        for h, hv in headers:
            req.add_header(h, hv)

    response = opener.open(req, post, timeout=timeout)
    link = response.read()
    response.close()
    return link


def get_decode(str, reg=None):
    if reg:
        str = re.findall(reg, str)[0]
    s1 = urllib.unquote(str[0: len(str) - 1]);
    t = '';
    for i in range(len(s1)):
        t += chr(ord(s1[i]) - s1[len(s1) - 1]);
    t = urllib.unquote(t)
    #    print t
    return t


def javascriptUnEscape(str):
    js = re.findall('unescape\(\'(.*?)\'', str)
    #    print 'js',js
    if (not js == None) and len(js) > 0:
        for j in js:
            # print urllib.unquote(j)
            str = str.replace(j, urllib.unquote(j))
    return str


def getEpocTime():
    import time
    return str(int(time.time() * 1000))


def getEpocTime2():
    import time
    return str(int(time.time()))
