Python string.count() Examples

def domain_counter(fqdn):
    ''' counter for domains based on the last two zones '''
    fulldata = []
    for each in fqdn:
        a = count((each),".")
        if a >= 2:
            fulldata.append(each[::-1])
    fulldata.sort()
    domain_trunk = []
    for each in fulldata:
        splitdata = each.split('.')
        # treat the first two as one
        TwoZones = str(splitdata[0])+"."+str(splitdata[1])
        #print TwoZones
        domain_trunk.append(TwoZones)        
    fulldata = []
    for each in domain_trunk:
        fulldata.append(each[::-1])
    return (Counter(fulldata)) 

def domain_counter(fqdn):
    ''' counter for domains based on the last two zones '''
    fulldata = []
    for each in fqdn:
        a = count((each),".")
        if a >= 2:
            fulldata.append(each[::-1])
    fulldata.sort()
    domain_trunk = []
    for each in fulldata:
        splitdata = each.split('.')
        # treat the first two as one
        TwoZones = str(splitdata[0])+"."+str(splitdata[1])
        #print TwoZones
        domain_trunk.append(TwoZones)        
    fulldata = []
    for each in domain_trunk:
        fulldata.append(each[::-1])
    return (Counter(fulldata)) 

def ip_print(sublist):
    query      = r.hget('IP:'+str(sublist), 'name')
    first      = r.hget('IP:'+str(sublist), 'first')
    date       = r.hget('IP:'+str(sublist), 'date')
    count      = r.hget('IP:'+str(sublist), 'count')
    rr_type    = r.hget('IP:'+str(sublist), 'type')
    ttl        = r.hget('IP:'+str(sublist), 'ttl')
    count      = r.hget('IP:'+str(sublist), 'count')
    #dns_client = r.hget(sublist, 'dns_client')
    #dns_server = r.hget(sublist, 'dns_server')
    #nss        = r.hget(sublist, 'nss')

    if first == None:
        first = '00000000'
    if date == None:
        date = '00000000'
    
    # PRINT FIELDS
    print "{0:18} {1:35} {2:9} {3:9} {4:8} {5:8} {6:6}".format(sublist,query,first[:8],date[:8],record_translate(rr_type),ttl,count)
    pass 

def ParseCSVLine(line):
    line = line.strip()
    values = []
    while line.__len__() > 0:
        if string.count(line, ',') > 0:
            if line[0] == '"':
                line = line[1:]
                values.append(line[:string.find(line, '"')])
                line = line[string.find(line, '"') + 1:]
                line = line[1:]
                if line.__len__() == 0:
                    values.append("")
            else:
                values.append(line[:string.find(line, ',')])
                line = line[string.find(line, ',') + 1:]
                if line.__len__() == 0:
                    values.append("")
        else:
            values.append(line)
            line = ""
    return values 

def _getFragWord(frags,maxWidth):
    ''' given a fragment list return a list of lists
        [size, spaces, (f00,w00), ..., (f0n,w0n)]
        each pair f,w represents a style and some string
    '''
    W = []
    n = 0
    s = 0
    for f in frags:
        text = f.text[:]
        W.append((f,text))
        cb = getattr(f,'cbDefn',None)
        if cb:
            _w = getattr(cb,'width',0)
            if hasattr(_w,'normalizedValue'):
                _w._normalizer = maxWidth
        n = n + stringWidth(text, f.fontName, f.fontSize)

        #s = s + _countSpaces(text)
        s = s + string.count(text, ' ') # much faster for many blanks

        #del f.text # we can't do this until we sort out splitting
                    # of paragraphs
    return n, s, W 

def translateDescReturn(desc_return):
    buff = ""
    for elem in desc_return.split(" "):
        tab = ""
        if "[" in elem:
            tab = "[" * string.count(elem, "[")
            elem = elem[:tab.find("[") - 2]

        if elem in BASIC_TYPES:
            buff += tab + BASIC_TYPES[elem] + " "
        else:
            if elem in ADVANCED_TYPES:
                buff += tab + ADVANCED_TYPES[elem] + " "
            else:
                if "." in elem:
                    buff += tab + "L" + elem.replace(".", "/") + "; "

    buff = buff[:-1]
    return buff 

def translateDescParams(desc_params):
    desc_params = desc_params.replace(" ", "")
    buff = ""

    for elem in desc_params.split(","):
        if elem != "":
            tab = ""
            if "[" in elem:
                tab = "[" * string.count(elem, "[")

                elem = elem[:tab.find("[") - 2]

            if elem not in BASIC_TYPES:
                if elem in ADVANCED_TYPES:
                    buff += tab + ADVANCED_TYPES[elem] + " "
                else:
                    buff += tab + "L" + elem.replace(".", "/") + "; "
            else:
                buff += tab + BASIC_TYPES[elem] + " "

    buff = buff[:-1]
    return buff 

def count_words(candidate_text, common_words=frequency.common_words['english'], case_sensitive=True):
   '''
   Count the instances of common words in the expected plaintext
   language, return the total number of characters matched in each
   word 

   candidate_text - (string) Sample to analyze
   common_words - (list) Sequences expected to appear in the text
   case_sensitive - (bool) Whether or not to match case sensitively
   '''
   score = 0

   for word in common_words:
      if not case_sensitive:
         word = word.lower()
      num_found = candidate_text.count(word)
      if num_found > 0:
         score += num_found * len(word)
      
   return score 

def count(self, domain):
        ''' count the subdomains within a ROOT domains ('domain.com') '''
        alldom = r.keys('Domain:'+str(domain))
        # call domain_counter function
        x = domain_counter(alldom).most_common(500)
        for k, v in x:
            print u'{0}: {1}'.format(k, v) 

def ip_flux(self, arg1):
        ''' Search IP space for a count of domains '''
        all_ips = r.keys('IP:'+str(arg1))
        allcount = []
        for each in all_ips:
            acount = r.hget(each, 'name') # working list
            #print acount
            if acount == None:
                pass
            else:
                allcount.append(acount)
        x = Counter(allcount).most_common(50)
        for k in x:
            print u'{0}'.format(k) 

def acount(self, domain):
        ''' counts of counts, or 'hits' for the domains in order, *.google.com or *.com are examples '''
        alldom = r.keys('Domain:'+str(domain)) 
        allcount = []
        for each in alldom:
            acount = r.hget(each, 'count')  
            if acount == None:
                pass
            else:
                allcount.append([int(acount),each]) 
        allcount.sort()
        for each in allcount:
            print str(each[0]),str(each[1]) 

def header_dom():
    ''' header and linked to domain searched output'''
    print "{0:40} {1:15} {2:9} {3:9} {4:5} {5:5} {6:8} {7:10} {8:8} {9:8} {10:9} {11:12}".format ("Domain","ips","first","date","rr","ttl","count","threat","date","tag","source","dom")
    pass 

def help(self):
        print 'pDNS2 commands\n'
        print 'DOMAIN EXAMPLES'
        print 'domain(\'*xstamper.com\') seeks all domains that end with xstamper.com'
        print 'ttl(\'0\')                use a number like 0 or 100 to get all the TTL of a specific value search is based on domain not IP'
        print 'count(\'*\')             return by query,count sub domains for a given domain example: .cn .com .google.com '
        print 'acount(\'*\')            return by query, counts of counts (usage), or \'hits\' for the domains in order, *.google.com or *.com are examples '
        print ''
        print 'IP EXAMPLES'
        print 'ip(\'222.*\')            returns 222.* or anything '
        print 'local()                  search entire database local resolved IP addresses that resolve to 127.0.0.1 etc. '
        print 'ip_flux(\'*.com\')       return a COUNT of domains in the IP space for each instance of a domain, use with ip_reverse'
        print 'x.ip_reverse(\'*\',\'seattletimes.com\') use with ip_flux, enumerate domains in the IP space'
        print ''
        print 'TAGGING'
        print 'tag_domain(\'ms.justice.cz\')    tag a domain'
        print 'tag_ip(\'194.213.41.92\')        tag an IP address '
        print ''        
        print 'THREAT DETECTION'
        print 'ip_sniff(\'192.168.1.1\')    search the domain space for a specific IP address, different then searching by IP '
        print 'date(\'20130101\')           return all records by date'

        print ''
        print 'ADMINISTRATIVE'
        print 'delete_key(\'Domain:*delete*\') Dangerous command, deletes a key, must use the entire key such as Domain: or IP:'
        print 'raw_record(\'Domain:xalrbngb-0.t.nessus.org\') view the raw record properties (no wildcards) use full key name'
        print 'pDNS2 tracks current state and last known, it is a snapshot of organization perception, not a log.'


# REDIS CONNECTION 

def header_ip():
    # header for IP based print information
    print  "{0:18} {1:35} {2:9} {3:9} {4:8} {5:8} {6:6} ".format ("IP","query","first","date","rr","ttl","count")
    pass 

def header_dom():
    ''' header and linked to domain searched output'''
    print "{0:40} {1:15} {2:9} {3:9} {4:5} {5:5} {6:9}".format ("Domain","ips","first","date","rr","ttl","count")
    pass 

def help(self):
        print 'pDNS2 commands\n'
        print 'DOMAIN EXAMPLES'
        print 'domain(\'*xstamper.com\') seeks all domains that end with xstamper.com'
        print 'ttl(\'0\')                use a number like 0 or 100 to get all the TTL of a specific value search is based on domain not IP'
        print 'tdom(\'*\')              return by query, tagged domain threats '
        print 'count(\'*\')             return by query,count sub domains for a given domain example: .cn .com .google.com '
        print 'acount(\'*\')            return by query, counts of counts (usage), or \'hits\' for the domains in order, *.google.com or *.com are examples '
        print ''
        print 'IP EXAMPLES'
        print 'ip(\'222.*\')            returns 222.* or anything '
        print 'tip(\'192.168.*\')       return by query, tagged IP addresses'
        print 'local()                  search entire database local resolved IP addresses that resolve to 127.0.0.1 etc. '
        print 'ip_flux(\'*.com\')       return a COUNT of domains in the IP space for each instance of a domain, use with ip_reverse'
        print 'x.ip_reverse(\'*\',\'seattletimes.com\') use with ip_flux, enumerate domains in the IP space'
        print ''
        print 'TAGGING'
        print 'tag_domain(\'ms.justice.cz\')    tag a domain'
        print 'tag_ip(\'194.213.41.92\')        tag an IP address '
        print ''        
        print 'THREAT DETECTION'
        print 'ip_sniff(\'192.168.1.1\')    search the domain space for a specific IP address, different then searching by IP '
        print 'date(\'20130101\')           return all records by date'
        print 'knn_dom(\'*.google.com\',20)  DO NOT USE IN DEV, returns euclidian distance threats by IP address '

        print ''
        print 'ADMINISTRATIVE'
        print 'delete_key(\'Domain:*delete*\') Dangerous command, deletes a key, must use the entire key such as Domain: or IP:'
        print 'raw_record(\'Domain:xalrbngb-0.t.nessus.org\') view the raw record properties (no wildcards) use full key name'
        print 'dom_unanswered(\'*.com\',1000) return unanswered queries by minimual count such as everything in .com with 1000 or more'
        print 'threat(5)                    search threats by provided date (DO NOT USE, work in progress)\n'        
        print 'pDNS2 tracks current state and last known, it is a snapshot of organization perception, not a log.'
        
        #x.rrecord('*') 

def count(self, domain):
        ''' count the subdomains within a ROOT domains ('domain.com') '''
        # consider an exclusion list
        # locate mass number of sub domains
        # REVERSE OF IP will resolve only a single domain, not all domains that have the same pairing
        # remember a change over time is more valuable then instant stats
        alldom = r.keys('Domain:'+str(domain))
        # call domain_counter function
        x = domain_counter(alldom).most_common(500)
        for k, v in x:
            print u'{0}: {1}'.format(k, v) 

def dom_unanswered(self, arg1,arg2):
        ''' find unaswered name request for searched domain space
        MX record are excluded
        format 'domain.com', min count'''
        all_dom = r.keys('Domain:'+str(arg1))
        print header_dom()
        for each in all_dom:
            work_ip = r.hget(each, 'ip')
            if work_ip == '':
                record = r.hget(each, 'type')
                count = r.hget(each, 'count')
                if record != '0x000f' and record != '0x0006' and int(count) >= arg2:
                    dom_print(each[7:]) 

def ip_flux(self, arg1):
        ''' Search IP space for a count of domains '''
        all_ips = r.keys('IP:'+str(arg1))
        allcount = []
        for each in all_ips:
            acount = r.hget(each, 'name') # working list
            #print acount
            if acount == None:
                pass
            else:
                allcount.append(acount)
        x = Counter(allcount).most_common(50)
        for k in x:
            print u'{0}'.format(k) 

def main():
    parser = argparse.ArgumentParser(description='pDNS2 a tool to collect and store DNS request and responses. Requires Redis server running on default port.')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-d','--domain')#, nargs=1
    group.add_argument('-i','--ip')
    parser.add_argument('-date','--date')
    parser.add_argument('-ips','--ip_sniff')
    parser.add_argument('-ttl','--ttl')
    parser.add_argument('-rr','--rrecord')
    parser.add_argument('-l','--local', action='store_true')
    parser.add_argument('-ac','--acount')
    parser.add_argument('-c','--count')
    parser.add_argument('-ipf','--ip_flux')
    parser.add_argument('-ipr','--ip_reverse', nargs=2)
    #group.add_argument('-v', '--verbose', action='store_true')
    args = parser.parse_args()

    #print args.domain
    if args.domain != None:
        x.domain(args.domain)
    elif args.ip != None:
        x.ip(args.ip)
    elif args.date != None:
        x.date(args.date)
    elif args.ip_sniff != None:
        x.ip_sniff(args.ip_sniff)
    elif args.ttl != None:
        x.ttl(args.ttl)
    elif args.rrecord != None:
        x.rrecord(args.rrecord)
    elif args.local != None:
        x.acount(args.local)
    elif args.count != None:
        x.count(args.count)
    elif args.ip_flux != None:
        x.ip_flux(args.ip_flux)
    elif args.ip_reverse != None:
        x.ip_reverse(args.ip_reverse[0],args.ip_reverse[1])
    else:
        x.help()
    sys.exit(-1) 

def ip_print(sublist):
    # split and remove the header before you get here along with sort
    # BASIC DNS / IP PROPERTIES
    query      = r.hget('IP:'+str(sublist), 'name')
    first      = r.hget('IP:'+str(sublist), 'first')
    date       = r.hget('IP:'+str(sublist), 'date')
    count      = r.hget('IP:'+str(sublist), 'count')
    rr_type    = r.hget('IP:'+str(sublist), 'type')
    ttl        = r.hget('IP:'+str(sublist), 'ttl')
    count      = r.hget('IP:'+str(sublist), 'count')
    threat      = r.hget('TIP:'+str(sublist), 'threat')
    evaldate   = r.hget('TIP:'+str(sublist), 'evaldate')
    tag       = r.hget('TIP:'+str(sublist), 'tag')
    source    = r.hget('TIP:'+str(sublist), 'source')
    dom_hit   = r.hget('TDOM:'+str(query), 'threat')

    ## below fields have value in some networks, you decide if you want to dispay it
    #dns_client = r.hget(sublist, 'dns_client')
    #dns_server = r.hget(sublist, 'dns_server')
    #nss        = r.hget(sublist, 'nss')

    
    # fix strings for dates
    if first == None:
        first = '00000000'
    if date == None:
        date = '00000000'
    
    
    # PRINT FIELDS
    print "{0:18} {1:35} {2:9} {3:9} {4:8} {5:8} {6:6} {7:12} {8:8} {9:12} {10:10} {11:12}".format(sublist,query,first[:8],date[:8],record_translate(rr_type),ttl,count,threat,evaldate,tag,source,dom_hit)
    pass 

def header_ip():
    # header for IP based print information
    print  "{0:18} {1:35} {2:9} {3:9} {4:8} {5:8} {6:6} {7:12} {8:8} {9:12} {10:10} {11:12}".format ("IP","query","first","date","rr","ttl","count","threat","date","tag","source","dom")
    pass 

def dom_print(sublist):
    ''' print the queried list in a 120 character line'''
    # sublist as passed contains the 'Domain:' with the search criteria
    ips        = r.hget('Domain:'+str(sublist), 'ip')
    first      = r.hget('Domain:'+str(sublist), 'first')
    date       = r.hget('Domain:'+str(sublist), 'date')
    rr_type    = r.hget('Domain:'+str(sublist), 'type')
    ttl        = r.hget('Domain:'+str(sublist), 'ttl')
    count      = r.hget('Domain:'+str(sublist), 'count')
    # access the TDOM keys
    threat    = r.hget('TDOM:'+str(sublist), 'threat')
    evaldate  = r.hget('TDOM:'+str(sublist), 'evaldate')
    tag       = r.hget('TDOM:'+str(sublist), 'tag')
    source    = r.hget('TDOM:'+str(sublist), 'source')   
    # check the subsequent key TIP if tagged as threat
    ips_hit   = r.hget('TIP:'+str(ips), 'threat')
    # fix strings for dates
    if first == None:
        first = '00000000'
    if date == None:
        date = '00000000'

    #FINAL PRINT
    print "{0:40} {1:15} {2:9} {3:9} {4:5} {5:5} {6:9} {7:10} {8:8} {9:8} {10:8} {11:12}".format(sublist,ips,first[:8],date[:8],record_translate(rr_type),ttl,count,threat, evaldate,tag,source,ips_hit)
    # denoting DNS client and server is sometime useful, depeends on the organization
    
    #dns_client = r.hget('Domain:'+str(sublist), 'dns_client')
    #dns_server = r.hget('Domain:'+str(sublist), 'dns_server')
    #print dns_client,",",sublist,",",dns_server,",",dns_server
    pass	


# ===================================== 

def unique(L):
	noDupli=[]
	[noDupli.append(i) for i in L if not noDupli.count(i)]
	return noDupli 

def getreadscount(self, mpileup, countmapped=False):
        # Input a mpileup result, which is a list
        # The count information is in the 5th column
        # count the number of mapped reads represented by ',' and '.' or, spliced reads
        # represented by the number of '>' and '<' in the string

        count = []

        # print "mpileup is a " + str(type(mpileup))

        if type(mpileup) is str:

            mpileupline = mpileup.split('\n')

            for itm in mpileupline:
                tmp = itm.split('\t')

                if countmapped:
                    if len(tmp) == 6:
                        count.append(tmp[0] + '\t' + tmp[1] + '\t' + self.countmapped(tmp[4]))
                else:
                    if len(tmp) == 6:
                        count.append(tmp[0] + '\t' + tmp[1] + '\t' + self.countspliced(tmp[4]))

        elif type(mpileup) is list:

            for itm in mpileup:

                if countmapped:
                    if len(mpileup) == 6:
                        count.append(itm[0] + '\t' + itm[1] + '\t' + self.countmapped(itm[4]))
                else:
                    if len(mpileup) == 6:
                        count.append(itm[0] + '\t' + itm[1] + '\t' + self.countspliced(itm[4]))

        return count 

def countspliced(self, string):
        # return the count of '>' and '<'
        rs = string.count('>') + string.count('<')
        return str(rs) 

def countmapped(self, string):
        # This function takes the 5th column (a string with mapping information),
        # and return the count of '.' and ','. Which are mapped read count

        rs = string.count(',') + string.count('.')

        return str(rs) 

def LoadLocations(path):
    global locations
    with open(path) as f:
        count = 0
        for line in f:
            count += 1
            if count < 3:
                continue
            (locId, country, region, city, postalCode, latitude, longitude,
            metroCode, areaCode) = ParseCSVLine(line)
            locations[int(locId)] = {"country": country, "region": region,
                "city": city, "postalCode": postalCode, "latitude": latitude,
                "longitude": longitude, "metroCode": metroCode,
                "areaCode": areaCode} 

def LoadBlocks(path):
    global blocks
    with open(path) as f:
        count = 0
        for line in f:
            count += 1
            if count < 3:
                continue
            blocks.append(ParseCSVLine(line))
    blocks = sorted(blocks, key=lambda it: int(it[0])) 

def _countSpaces(text):
    return string.count(text, ' ')
##  i = 0
##  s = 0
##  while 1:
##      j = string.find(text,' ',i)
##      if j<0: return s
##      s = s + 1
##      i = j + 1