Python os.path.append() Examples

The following are code examples for showing how to use os.path.append(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the ones you don't like. You can also save this page to your account.

Example 1
Project: farfetchd   Author: isislovecruft   File: server.py    (license) View Source Project 6 votes vote down vote up
def render_GET(self, request):
        request.responseHeaders.setRawHeaders(b"Content-Type", ["text/html"])

        data = bytes()
        path = []

        path.append(os.path.dirname(__file__))

        # If we're being run from the build/ directory of a source or git tree,
        # append the full path:
        if not os.path.isabs(path[0]):
            path.insert(0, os.getcwd())
        # If we're being run as part of some unittests, get rid of the test dir:
        if path[0].endswith("_trial_temp"):
            path[0] = path[0].rsplit("_trial_temp")[0]

        path.append('API.html')
        spec = os.path.sep.join(path)

        with open(spec) as fh:
            data += bytes(fh.read())

        return data 
Example 2
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 6 votes vote down vote up
def check_if_match():
    diff = 0
    bad_chars = defaultdict(list)
    minlen = min(len(buffers[0]), len(buffers[1]))

    for i in xrange(minlen):
        if buffers[0][i] != buffers[1][i]:
            diff += 1
            bad_chars[buffers[0][i]].append(buffers[1][i])

    if len(buffers[0]) > minlen:
        bad_chars[-1].append(buffers[1][-1])
    elif len(buffers[1]) > minlen:
        bad_chars[-1].append(buffers[0][-1])

    return (diff, bad_chars) 
Example 3
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 6 votes vote down vote up
def deserialize_lms_sig(buffer):
    q = deserialize_u32(buffer[0:4])
    # print "q: " + str(q)
    lmots_type = typecode_peek(buffer[4:8])
    # print "lmots_type: " + str(lmots_type)
    if lmots_type in lmots_params:
        pos = 4 + LmotsSignature.bytes(lmots_type)
    else:
        raise ValueError(err_unknown_typecode, str(lmots_type))
    lmots_sig = buffer[4:pos]
    lms_type = typecode_peek(buffer[pos:pos+4])
    if lms_type in lms_params:
        m, h, LenI = lms_params[lms_type]
    else:
        raise ValueError(err_unknown_typecode, str(lms_type))
    if (q >= 2**h):
        raise ValueError(err_bad_value)
    pos = pos + 4
    path = list()
    for i in xrange(0, h):
        path.append(buffer[pos:pos+m])
        pos = pos + m
    # PrintUtl.print_hex("buffer tail", buffer[pos:])
    return lms_type, q, lmots_sig, path 
Example 4
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 6 votes vote down vote up
def sign(self, message):
        while (self.prv[-1].is_exhausted()):
            print "level " + str(len(self.prv)) + " is exhausted"
            if (len(self.prv) == 1):
                raise ValueError(err_private_key_exhausted)
            self.prv.pop()
            self.pub.pop()
            self.sig.pop()
        while (len(self.prv) < self.levels):
            print "refreshing level " + str(len(self.prv))
            self.prv.append(LmsPrivateKey(lms_type=self.prv[0].lms_type, lmots_type=self.prv[0].lmots_type))
            self.pub.append(self.prv[-1].get_public_key())
            self.sig.append(self.prv[-2].sign(self.pub[-1].serialize()))            

        # sign message 
        lms_sig = self.prv[-1].sign(message)
        return serialize_hss_sig(self.levels-1, self.pub, self.sig, lms_sig) 
Example 5
Project: Entity-Triple-Entity-Extraction   Author: shubham0420   File: Train.py    (license) View Source Project 6 votes vote down vote up
def Dependency_Parser(Sentence):

    try:
        Dep_Parser_Output = Dep_Parser(Sentence)
        Dep_Parser_Tags = []
        
        for Token in Dep_Parser_Output:
            Dep, Head, Orth = Token.dep_, Token.head.orth_, Token.orth_
            if Dep == u'ROOT':
                Dep = u'root'
                Head = u'ROOT'
                
            Dep_Parser_Tags.append([Dep, Head, Orth])
        return Dep_Parser_Tags 
    
    except Exception as e :
        print e
        return False 
Example 6
Project: Entity-Triple-Entity-Extraction   Author: shubham0420   File: Train.py    (license) View Source Project 6 votes vote down vote up
def Save_Wiki_IDs(self):
        
        if Create_Folder(Path_Directory + '/Data/Wiki_IDs/', Flag_Remove = False):

            Sub_Classes = []
            Wiki_IDs_File = io.open(Path_Directory + '/Data/Wiki_IDs/{}'.format(self.Class), 'w', encoding = 'utf8')
        
            Results_Sub_Classes = SPARQL_Commands(Query_Subclasses % self.Class)
            if Results_Sub_Classes:   
                for Result in Results_Sub_Classes["results"]["bindings"]:
                    Sub_Classes.append(Result["x"]["value"])
                        
            print 'Accessed Class: {}'.format(self.Class)
            print 'No. of Sub Classes: {}'.format(len(Sub_Classes))
            
            for Subclass in Sub_Classes:
                print Subclass
                Results_Wiki_ID =  SPARQL_Commands(Query_Wiki_ID % '<{}>'.format(Subclass))
                if Results_Wiki_ID:
                    for Result in Results_Wiki_ID["results"]["bindings"]:
                        Wiki_IDs_File.write(unicode(Result["wikiid"]["value"]))
                        Wiki_IDs_File.write(u'\n')
           
            Wiki_IDs_File.close() 
Example 7
Project: reahl   Author: reahl   File: eggs.py    (license) View Source Project 6 votes vote down vote up
def search(self, from_vertex):
        self.discovered[from_vertex] = True
        self.entered[from_vertex] = self.count
        self.count += 1
        for i in self.graph[from_vertex]:
            if i not in self.discovered:
                self.parents[i] = from_vertex
                self.search(i)
            elif self.entered[i] < self.entered[from_vertex] and i not in self.exited:
                raise CircularDependencyDetected(self.path(i, from_vertex)+[i])
            elif i not in self.parents:
                self.parents[i] = from_vertex

        self.exited[from_vertex] = self.count
        self.count += 1
        self.topological_order.append(from_vertex) 
Example 8
Project: satpy   Author: pytroll   File: hdfeos_l1b.py    (license) View Source Project 6 votes vote down vote up
def load(self, keys, interpolate=True, raw=False):
        projectables = []
        for key in keys:
            dataset = self.sd.select(key.name.capitalize())
            fill_value = dataset.attributes()["_FillValue"]
            try:
                scale_factor = dataset.attributes()["scale_factor"]
            except KeyError:
                scale_factor = 1
            data = np.ma.masked_equal(dataset.get(), fill_value) * scale_factor

            # TODO: interpolate if needed
            if key.resolution is not None and key.resolution < self.resolution and interpolate:
                data = self._interpolate(data, self.resolution, key.resolution)
            if raw:
                projectables.append(data)
            else:
                projectables.append(Dataset(data, id=key))

        return projectables 
Example 9
Project: nodeembedding-to-communityembedding   Author: andompesta   File: graph_utils.py    (license) View Source Project 6 votes vote down vote up
def build_deepwalk_corpus(G, num_paths, path_length, alpha=0, rand=random.Random(0)):
    '''
    extract the walks form the graph used for context embeddings
    :param G: graph
    :param num_paths: how many random walks to form a sentence
    :param path_length: how long each path -> length of the sentence
    :param alpha: restart probability
    :param rand: random function
    :return:
    '''
    walks = []
    nodes = list(G.nodes())
    for cnt in range(num_paths):
        rand.shuffle(nodes)
        for node in nodes:
            walks.append(__random_walk__(G, path_length, rand=rand, alpha=alpha, start=node))
    return np.array(walks) 
Example 10
Project: memex-dossier-open   Author: dossier   File: routes.py    (license) View Source Project 6 votes vote down vote up
def v1_subtopic_list(request, response, kvlclient, fid, sfid):
    '''Retrieves a list of items in a subfolder.

    The route for this endpoint is:
    ``GET /dossier/v1/folder/<fid>/subfolder/<sfid>``.

    (Temporarily, the "current user" can be set via the
    ``annotator_id`` query parameter.)

    The payload returned is a list of two element arrays. The first
    element in the array is the item's content id and the second
    element is the item's subtopic id.
    '''
    path = urllib.unquote(fid) + '/' + urllib.unquote(sfid)
    try:
        items = []
        for it in new_folders(kvlclient, request).list(path):
            if '@' in it.name:
                items.append(it.name.split('@'))
            else:
                items.append((it.name, None))
        return items
    except KeyError:
        response.status = 404
        return [] 
Example 11
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 5 votes vote down vote up
def normalize_input(self):
        input = []
        for line in self.input.split('\n'):
            line = line.strip()
            line2 = line.encode('string-escape')
            input.append(line2)
        self.input = '\n'.join(input) 
Example 12
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 5 votes vote down vote up
def post_process_bytes_line(line):
        outb = []
        l = line.strip()[:]
        strip = ['0x', ',', ' ', '\\', 'x', '%u', '+', '.', "'", '"']
        for s in strip:
            l = l.replace(s, '')

        for i in xrange(0, len(l), 2):
            outb.append(int(l[i:i+2], 16))
        return outb 
Example 13
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 5 votes vote down vote up
def draw_chunk_table(comp):
    ''' Outputs a table that compares the found memory chunks side-by-side
    in input file vs. memory '''
    table = [('', '', '', '', 'File', 'Memory', 'Note')]
    delims = (' ', ' ', ' ', ' | ', ' | ', ' | ', '')
    last_unmodified = comp.get_last_unmodified_chunk()
    for c in comp.get_chunks():
        if   c.dy == 0:    note = 'missing'
        elif c.dx > c.dy:  note = 'compacted'
        elif c.dx < c.dy:  note = 'expanded'
        elif c.unmodified: note = 'unmodified!'
        else:              note = 'corrupted'
        table.append((c.i, c.j, c.dx, c.dy, shorten_bytes(c.xchunk), shorten_bytes(c.ychunk), note))

    # draw the table
    sizes = tuple(max(len(str(c)) for c in col) for col in zip(*table))
    for i, row in enumerate(table):
        out('\t' + ''.join(str(x).ljust(size) + delim for x, size, delim in zip(row, sizes, delims)))
        if i == 0 or (i == last_unmodified + 1 and i < len(table)):
            out('\t' + '-' * (sum(sizes) + sum(len(d) for d in delims)))


#
# Memory comparison algorithm originally taken from Mona.py by Peter Van Eeckhoutte - Corelan GCV
# https://github.com/corelan/mona
#
# It utilizes modified Longest Common Subsequence algorithm to mark number of modifications over
# supplied input to let it be transformed into another input, as compared to.
# 
Example 14
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 5 votes vote down vote up
def get_grid(self):
        ''' Builds a 2-d suffix grid for our DP algorithm. '''
        x = self.x
        y = self.y[:len(x)*2]
        width, height  = len(x), len(y)
        values = [[0] * (width + 1) for j in range(height + 1)]
        moves  = [[0] * (width + 1) for j in range(height + 1)]
        equal  = [[x[i] == y[j] for i in range(width)] for j in range(height)]
        equal.append([False] * width)

        for j, i in itertools.product(rrange(height + 1), rrange(width + 1)):
            value = values[j][i]
            if i >= 1 and j >= 1:
                if equal[j-1][i-1]:
                    values[j-1][i-1] = value + 1
                    moves[j-1][i-1] = 2
                elif value > values[j][i-1]:
                    values[j-1][i-1] = value
                    moves[j-1][i-1] = 2
            if i >= 1 and not equal[j][i-1] and value - 2 > values[j][i-1]:
                values[j][i-1] = value - 2
                moves[j][i-1] = 1
            if i >= 1 and j >= 2 and not equal[j-2][i-1] and value - 1 > values[j-2][i-1]:
                values[j-2][i-1] = value - 1
                moves[j-2][i-1] = 3
        return (values, moves) 
Example 15
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 5 votes vote down vote up
def construct_comparator_dump(self, mapping):
        def toprint(x, src):
            c = x
            if len(c) == 0: c = ' '
            elif len(c) == 2: c = x[1]

            if ord(c) >= 0x20 and ord(c) < 0x7f:
                return c
            else: 
                return '.'

        for i, chunk in enumerate(HexDumpPrinter.extract_chunks(mapping)):
            chunk = list(chunk)  # save generator result in a list
            src, mapped = zip(*chunk)
            values = []
            for left, right in zip(src, mapped):
                if   left == right:   values.append('')             # byte matches original
                elif len(right) == 0: values.append('-1')           # byte dropped
                elif len(right) == 2: values.append('+1')           # byte expanded
                else:                 values.append(bin2hex(right)) # byte modified

            line1 = '%04x' % (i * 16) + ' | ' + bin2hex(src).ljust(49, ' ')
            line2 = '%04x' % (i * 16) + ' | ' + ' '.join(sym.ljust(2, self.fill_matching) for sym in values)

            line1 += '| ' + ''.join(map(lambda x: x if ord(x) >= 0x20 and ord(x) < 0x7f else '.', src)).ljust(16, ' ')
            ascii2 = '| '
            for i in range(len(values)): ascii2 += toprint(values[i], src[i])
            for i in range(len(values), 16): ascii2 += ' '
            line2 = line2.ljust(56, ' ')
            line2 += ascii2

            #out(dbg("Line1: ('%s')" % line1))
            #out(dbg("Line2: ('%s')" % line2))

            self.dump1.append(line1)
            self.dump2.append(line2) 
Example 16
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 5 votes vote down vote up
def extract_bytes(line):
        linet = line.split(' | ')
        strbytes = [linet[1][i:i+2] for i in range(0, len(linet[1]), 3)]
        bytes = []
        for s in strbytes:
            bytes.append(s)
        return bytes 
Example 17
Project: OSPTF   Author: xSploited   File: PupyClient.py    (license) View Source Project 5 votes vote down vote up
def get_packages_path(self):
        """ return the list of path to search packages for depending on client OS and architecture """
        path=[]
        if self.is_windows():
            if self.is_proc_arch_64_bits():
                path.append(os.path.join(ROOT, "packages","windows","amd64"))
                path.append(os.path.join("packages","windows","amd64"))
            else:
                path.append(os.path.join(ROOT, "packages","windows","x86"))
                path.append(os.path.join("packages","windows","x86"))

            path.append(os.path.join(ROOT, "packages","windows","all"))
            path.append(os.path.join("packages","windows","all"))
        elif self.is_unix():
            if self.is_proc_arch_64_bits():
                path.append(os.path.join(ROOT, "packages","linux","amd64"))
                path.append(os.path.join("packages","linux","amd64"))
            else:
                path.append(os.path.join(ROOT, "packages","linux","x86"))
                path.append(os.path.join("packages","linux","x86"))

            path.append(os.path.join(ROOT, "packages","linux","all"))
            path.append(os.path.join("packages","linux","all"))
        if self.is_android():
            path.append(os.path.join(ROOT, "packages","android"))
            path.append(os.path.join("packages","android"))

        path.append(os.path.join(ROOT, "packages","all"))
        path.append(os.path.join("packages","all"))
        return path 
Example 18
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def deserialize(cls, buffer):
        lmots_type = typecode_peek(buffer[0:4])
        if lmots_type in lmots_params:
            n, p, w, ls = lmots_params[lmots_type]
        else:
            raise ValueError(err_unknown_typecode, str(lmots_type))
        if (len(buffer) != cls.bytes(lmots_type)):
            raise ValueError(err_bad_length)
        C = buffer[4:n+4]
        y = list()
        pos = n+4
        for i in xrange(0, p):
            y.append(buffer[pos:pos+n])
            pos = pos + n
        return cls(C, y, lmots_type) 
Example 19
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, S=None, SEED=None, lmots_type=lmots_sha256_n32_w8):   
        n, p, w, ls = lmots_params[lmots_type]
        if S is None:
            self.S = entropySource.read(n)
        else:
            self.S = S
        self.x = list()
        if SEED is None:
            for i in xrange(0, p):
                self.x.append(entropySource.read(n))
        else:
            for i in xrange(0, p):
                self.x.append(H(self.S + SEED + u16str(i+1) + D_PRG))
        self.type = lmots_type
        self._signatures_remaining = 1 
Example 20
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def sign(self, message):
        if self._signatures_remaining != 1:
            raise ValueError(err_private_key_exhausted)
        n, p, w, ls = lmots_params[self.type]
        C = entropySource.read(n) 
        hashQ = H(self.S + C + message + D_MESG)
        V = hashQ + checksum(hashQ, w, ls)
        y = list()
        for i, x in enumerate(self.x):
            tmp = x
            for j in xrange(0, coef(V, i, w)):
                tmp = H(self.S + tmp + u16str(i) + u8str(j) + D_ITER)
            y.append(tmp)
        self._signatures_remaining = 0
        return LmotsSignature(C, y, self.type).serialize() 
Example 21
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def get_param_list(cls):
        param_list = list()
        for t in lmots_params.keys():
            param_list.append({'lmots_type':t})
        return param_list 
Example 22
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def get_path(self, node_num):
        path = list()
        while node_num > 1:
            if (node_num % 2):
                path.append(self.nodes[node_num - 1])
            else:
                path.append(self.nodes[node_num + 1])
            node_num = node_num/2
        return path 
Example 23
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def get_param_list(cls):
        param_list = list()
        for x in lmots_params.keys():
            for y in lms_params.keys():
                param_list.append({'lmots_type':x, 'lms_type':y})
        return param_list 
Example 24
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def deserialize_hss_sig(buffer):
    hss_max_levels = 8
    levels = deserialize_u32(buffer[0:4]) + 1
    if (levels > hss_max_levels):
        raise ValueError(err_bad_value)
    siglist = list()
    publist = list()
    tmp = buffer[4:]
    for i in xrange(0, levels-1):
        lms_sig, tmp = parse_lms_sig(tmp)
        siglist.append(lms_sig)
        lms_pub, tmp = LmsPublicKey.parse(tmp)
        publist.append(lms_pub)
    msg_sig = tmp
    return levels, publist, siglist, msg_sig 
Example 25
Project: hash-sigs   Author: davidmcgrew   File: hss.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, levels=2, lms_type=lms_sha256_m32_h5, lmots_type=lmots_sha256_n32_w8, prv0=None):
        self.levels = levels
        self.prv = list()
        self.pub = list()
        self.sig = list()
        if prv0 is None:
            prv0 = LmsPrivateKey(lms_type=lms_type, lmots_type=lmots_type)
        self.prv.append(prv0)
        self.pub.append(self.prv[0].get_public_key())
        for i in xrange(1, self.levels):
            self.prv.append(LmsPrivateKey(lms_type=lms_type, lmots_type=lmots_type))
            self.pub.append(self.prv[-1].get_public_key())
            self.sig.append(self.prv[-2].sign(self.pub[-1].serialize())) 
Example 26
Project: pupy   Author: ru-faraon   File: PupyClient.py    (license) View Source Project 5 votes vote down vote up
def get_packages_path(self):
        """ return the list of path to search packages for depending on client OS and architecture """
        path=[]
        if self.is_windows():
            if self.is_proc_arch_64_bits():
                path.append(os.path.join(ROOT, "packages","windows","amd64"))
                path.append(os.path.join("packages","windows","amd64"))
            else:
                path.append(os.path.join(ROOT, "packages","windows","x86"))
                path.append(os.path.join("packages","windows","x86"))

            path.append(os.path.join(ROOT, "packages","windows","all"))
            path.append(os.path.join("packages","windows","all"))
        elif self.is_unix():
            if self.is_proc_arch_64_bits():
                path.append(os.path.join(ROOT, "packages","linux","amd64"))
                path.append(os.path.join("packages","linux","amd64"))
            else:
                path.append(os.path.join(ROOT, "packages","linux","x86"))
                path.append(os.path.join("packages","linux","x86"))

            path.append(os.path.join(ROOT, "packages","linux","all"))
            path.append(os.path.join("packages","linux","all"))
        if self.is_android():
            path.append(os.path.join(ROOT, "packages","android"))
            path.append(os.path.join("packages","android"))

        path.append(os.path.join(ROOT, "packages","all"))
        path.append(os.path.join("packages","all"))
        return path 
Example 27
Project: reahl   Author: reahl   File: eggs.py    (license) View Source Project 5 votes vote down vote up
def path(self, from_vertex, to_vertex):
        path = []
        i = to_vertex
        path.append(i)
        while i in self.parents and i is not from_vertex:
            i = self.parents[i]
            path.append(i)
        if i is not from_vertex:
            raise NoDependencyPathFound(from_vertex, to_vertex)
        path.reverse()
        return path 
Example 28
Project: reahl   Author: reahl   File: eggs.py    (license) View Source Project 5 votes vote down vote up
def compute_all_relevant_interfaces(cls, main_egg):
        interfaces = []

        for i in cls.compute_ordered_dependent_distributions(main_egg):
            entry_map = i.get_entry_map('reahl.eggs')
            if entry_map:
                classes = list(entry_map.values())
                assert len(classes) == 1, 'Only one eggdeb class per egg allowed'
                interfaces.append(classes[0].load()(i))


        return interfaces 
Example 29
Project: satpy   Author: pytroll   File: hdfeos_l1b.py    (license) View Source Project 5 votes vote down vote up
def read_mda(self, attribute):
        lines = attribute.split('\n')
        mda = {}
        current_dict = mda
        path = []
        for line in lines:
            if not line:
                continue
            if line == 'END':
                break
            key, val = line.split('=')
            key = key.strip()
            val = val.strip()
            try:
                val = eval(val)
            except NameError:
                pass
            if key in ['GROUP', 'OBJECT']:
                new_dict = {}
                path.append(val)
                current_dict[val] = new_dict
                current_dict = new_dict
            elif key in ['END_GROUP', 'END_OBJECT']:
                if val != path[-1]:
                    raise SyntaxError
                path = path[:-1]
                current_dict = mda
                for item in path:
                    current_dict = current_dict[item]
            elif key in ['CLASS', 'NUM_VAL']:
                pass
            else:
                current_dict[key] = val
        return mda 
Example 30
Project: PythonTraining   Author: sure-it   File: lkT4.py    (license) View Source Project 5 votes vote down vote up
def file_1(b):                      #????????
    file_list=[]
    file_name=os.listdir(b)         #????????????
    os.chdir(b)                     #?????????????????????
    new_path2=os.getcwd()           #??????
    for i in file_name:
        merge_path_and_file=os.path.join(new_path2,i)       #??????????????
        file_list.append(merge_path_and_file)       
    return file_list 
Example 31
Project: PythonTraining   Author: sure-it   File: lkT4.py    (license) View Source Project 5 votes vote down vote up
def input_path(a):                  #????????
    path=[]
    path2=file_1(a)                 #????????
    for o in path2:
        if os.path.isdir(o):        #????????
            input_path(o)           #   ??????????
            path.append(o)
        else:
            print(o)                #   ?????????????????
    return path 
Example 32
Project: nodeembedding-to-communityembedding   Author: andompesta   File: graph_utils.py    (license) View Source Project 5 votes vote down vote up
def __random_walk__(G, path_length, alpha=0, rand=random.Random(), start=None):
    '''
    Returns a truncated random walk.
    :param G: networkx graph
    :param path_length: Length of the random walk.
    :param alpha: probability of restarts.
    :param rand: random number generator
    :param start: the start node of the random walk.
    :return:
    '''

    if start:
        path = [start]
    else:
        # Sampling is uniform w.r.t V, and not w.r.t E
        path = [rand.choice(G.nodes)]

    while len(path) < path_length:
        cur = path[-1]
        if len(G.neighbors(cur)) > 0:
            if rand.random() >= alpha:
                path.append(rand.choice(G.neighbors(cur)))
            else:
                path.append(path[0])
        else:
            break
    return path 
Example 33
Project: nodeembedding-to-communityembedding   Author: andompesta   File: graph_utils.py    (license) View Source Project 5 votes vote down vote up
def write_walks_to_disk(G, filebase, num_paths, path_length, alpha=0, rand=random.Random(0), num_workers=cpu_count()):
    '''
    save the random walks on files so is not needed to perform the walks at each execution
    :param G: graph to walks on
    :param filebase: location where to save the final walks
    :param num_paths: number of walks to do for each node
    :param path_length: lenght of each walks
    :param alpha: restart probability for the random walks
    :param rand: generator of random numbers
    :param num_workers: number of thread used to execute the job
    :return:
    '''
    global __current_graph
    global __vertex2str
    __current_graph = G
    __vertex2str = {v:str(v) for v in G.nodes()}
    files_list = ["{}.{}".format(filebase, str(x)) for x in range(num_paths)]
    expected_size = len(G)
    args_list = []
    files = []

    if num_paths <= num_workers:
        paths_per_worker = [1 for x in range(num_paths)]
    else:
        paths_per_worker = [len(list(filter(lambda z: z!= None, [y for y in x]))) for x in grouper(int(num_paths / num_workers)+1, range(1, num_paths+1))]

    with ProcessPoolExecutor(max_workers=num_workers) as executor:
        for size, file_, ppw in zip(executor.map(count_lines, files_list), files_list, paths_per_worker):
            args_list.append((ppw, path_length, alpha, random.Random(rand.randint(0, 2**31)), file_))

    with ProcessPoolExecutor(max_workers=num_workers) as executor:
        for file_ in executor.map(_write_walks_to_disk, args_list):
            files.append(file_)

    return files 
Example 34
Project: GraphEmbeddingsRecommenderSystems   Author: himangshunits   File: graph.py    (license) View Source Project 5 votes vote down vote up
def make_undirected(self):
  
    t0 = time()

    for v in self.keys():
      for other in self[v]:
        if v != other:
          self[other].append(v)
    
    t1 = time()
    logger.info('make_directed: added missing edges {}s'.format(t1-t0))

    self.make_consistent()
    return self 
Example 35
Project: GraphEmbeddingsRecommenderSystems   Author: himangshunits   File: graph.py    (license) View Source Project 5 votes vote down vote up
def random_walk(self, path_length, alpha=0, rand=random.Random(), start=None):
    """ Returns a truncated random walk.

        path_length: Length of the random walk.
        alpha: probability of restarts.
        start: the start node of the random walk.
    """
    G = self
    if start:
      path = [start]
    else:
      # Sampling is uniform w.r.t V, and not w.r.t E
      path = [rand.choice(G.keys())]

    while len(path) < path_length:
      cur = path[-1]
      if len(G[cur]) > 0:
        if rand.random() >= alpha:
          path.append(rand.choice(G[cur]))
        else:
          path.append(path[0])
      else:
        break
    return path

# TODO add build_walks in here 
Example 36
Project: GraphEmbeddingsRecommenderSystems   Author: himangshunits   File: graph.py    (license) View Source Project 5 votes vote down vote up
def build_deepwalk_corpus(G, num_paths, path_length, alpha=0,
                      rand=random.Random(0)):
  walks = []

  nodes = list(G.nodes())
  
  for cnt in range(num_paths):
    rand.shuffle(nodes)
    for node in nodes:
      walks.append(G.random_walk(path_length, rand=rand, alpha=alpha, start=node))
  
  return walks 
Example 37
Project: GraphEmbeddingsRecommenderSystems   Author: himangshunits   File: graph.py    (license) View Source Project 5 votes vote down vote up
def load_edgelist(file_, undirected=True):
  G = Graph()
  with open(file_) as f:
    for l in f:
      x, y = l.strip().split()[:2]
      x = int(x)
      y = int(y)
      G[x].append(y)
      if undirected:
        G[y].append(x)
  
  G.make_consistent()
  return G 
Example 38
Project: GraphEmbeddingsRecommenderSystems   Author: himangshunits   File: graph.py    (license) View Source Project 5 votes vote down vote up
def from_numpy(x, undirected=True):
    G = Graph()

    if issparse(x):
        cx = x.tocoo()
        for i,j,v in zip(cx.row, cx.col, cx.data):
            G[i].append(j)
    else:
      raise Exception("Dense matrices not yet supported.")

    if undirected:
        G.make_undirected()

    G.make_consistent()
    return G 
Example 39
Project: memex-dossier-open   Author: dossier   File: routes.py    (license) View Source Project 5 votes vote down vote up
def make_path(fid, sfid=None, cid=None, subid=None):
    path = [urllib.unquote(fid)]
    if sfid is not None:
        path.append(urllib.unquote(sfid))
        if cid is not None:
            if subid is not None:
                path.append(cid + '@' + subid)
            else:
                path.append(cid)
    return '/'.join(path) 
Example 40
Project: memex-dossier-open   Author: dossier   File: routes.py    (license) View Source Project 5 votes vote down vote up
def add_link_headers(page, per):
        links = []
        # Add the "first" and "prev" links.
        if page > 1:
            links.append(('first', setqp('page', '1')))
            links.append(('prev', setqp('page', str(page - 1))))
        # We never really know when the stream ends, so there is always a
        # "next" link.
        links.append(('next', setqp('page', str(page + 1))))
        response.headers['Link'] = ', '.join(map(tuple_to_link, links)) 
Example 41
Project: chopsticks   Author: lordmauve   File: tunnel.py    (license) View Source Project 5 votes vote down vote up
def cmd_args(self):
        base = super(Docker, self).cmd_args()
        args = []
        if self.rm:
            args.append('--rm')

        return [
            'docker',
            'run',
            '-i',
            '--name',
            self.host,
        ] + args + [self.image] + base 
Example 42
Project: chopsticks   Author: lordmauve   File: tunnel.py    (license) View Source Project 5 votes vote down vote up
def cmd_args(self):
        args = ['ssh', '-o', 'PasswordAuthentication=no']
        if self.user:
            args.extend(['-l', self.user])
        args.append(self.host)
        if self.sudo:
            args.append('sudo')
        args.extend(super(SSHTunnel, self).cmd_args())
        return ['"%s"' % w if ' ' in w else w for w in args]


# An alias, because this is the default tunnel type 
Example 43
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 4 votes vote down vote up
def get_blocks(self):
        '''
            Compares two binary strings under the assumption that y is the result of
            applying the following transformations onto x:

             * change single bytes in x (likely)
             * expand single bytes in x to two bytes (less likely)
             * drop single bytes in x (even less likely)

            Returns a generator that yields elements of the form (unmodified, xdiff, ydiff),
            where each item represents a binary chunk with "unmodified" denoting whether the
            chunk is the same in both strings, "xdiff" denoting the size of the chunk in x
            and "ydiff" denoting the size of the chunk in y.

            Example:
            >>> x = "abcdefghijklm"
            >>> y = "mmmcdefgHIJZklm"
            >>> list(MemoryComparator(x, y).get_blocks())
            [(False, 2, 3), (True, 5, 5),
             (False, 3, 4), (True, 3, 3)]
        '''
        x, y = self.x, self.y
        _, moves = self.get_grid()

        # walk the grid
        path = []
        i, j = 0, 0
        while True:
            dy, dx = self.move_to_gradient[moves[j][i]]
            if dy == dx == 0: break
            path.append((dy == 1 and x[i] == y[j], dy, dx))
            j, i = j + dy, i + dx

        for i, j in zip(range(i, len(x)), itertools.count(j)):
            if j < len(y): path.append((x[i] == y[j], 1, 1))
            else:          path.append((False,        0, 1))

        i = j = 0
        for unmodified, subpath in itertools.groupby(path, itemgetter(0)):
            ydiffs = map(itemgetter(1), subpath)
            dx, dy = len(ydiffs), sum(ydiffs)
            yield unmodified, dx, dy
            i += dx
            j += dy 
Example 44
Project: expdevBadChars   Author: mgeeky   File: badchars.py    (license) View Source Project 4 votes vote down vote up
def reconstruct_line(letter, line, bytes):
        bytes_line = ''
        linet = line.split(' | ')
        color_address = False

        diff_indexes = []
        i = 0
        for b in bytes:
            if len(b) != 2:
                # difference
                diff_indexes.append(i)
                color_address = True
                if len(b) == 4:
                    # not colored difference
                    l = list(bytes_line)
                    if len(l) > 1:
                        l[-1] = b[0]
                    bytes_line = ''.join(l)
                    bytes_line += b[1:]
                else:
                    # colored difference
                    bytes_line += b + ' '
            else:
                bytes_line += b + ' '
            i += 1

        address = linet[0]
        ascii = linet[2]

        for b in range(len(bytes), 16):
            bytes_line += ' ' * 3
            ascii += ' '

        if options.colored:
            new_ascii = ''
            for j in range(len(ascii)):
                if j in diff_indexes:
                    new_ascii += bcolors.FAIL + ascii[j] + bcolors.ENDC
                else:
                    new_ascii += ascii[j]
            new_ascii, ascii = ascii, new_ascii

        if color_address or len(letter) > 1:
            if options.colored:
                address = bcolors.OKBLUE + address + bcolors.ENDC
            else:
                address = address

        return '{}{} | {} | {}'.format(letter, address, bytes_line, ascii.ljust(16)) 
Example 45
Project: GraphEmbeddingsRecommenderSystems   Author: himangshunits   File: graph.py    (license) View Source Project 4 votes vote down vote up
def load_movie_data():
    # Movie data files used for building the graph
    movies_directors_filename = "./data/movie_directors.dat"
    movies_actors_filename = "./data/movie_actors.dat"
    movies_genres_filename = "./data/movie_genres.dat"
    movies_filename = "./data/movies.dat"
    
    # Load the data about the movies into a dictionary
    # The dictionary maps a movie ID to a movie object
    # Also store the unique directors, actors, and genres
    movies = {}
    with open(movies_filename, "r") as fin:
        fin.next()  # burn metadata line
        for line in fin:
            m_id, name = line.strip().split()[:2]
            movies["m"+m_id] = Movie(name)
    
    directors = set([])
    with open(movies_directors_filename, "r") as fin:
        fin.next()  # burn metadata line
        for line in fin:
            m_id, director = line.strip().split()[:2]
            if "m"+m_id in movies:
                movies["m"+m_id].director = director
            directors.add(director)
    
    actors = set([])
    with open(movies_actors_filename, "r") as fin:
        fin.next()  # burn metadata line
        for line in fin:
            m_id, actor = line.strip().split()[:2]
            if "m"+m_id in movies:
                movies["m"+m_id].actors.append(actor)
            actors.add(actor)
    
    genres = set([])
    with open(movies_genres_filename, "r") as fin:
        fin.next()  # burn metadata line
        for line in fin:
            m_id, genre = line.strip().split()
            if "m"+m_id in movies:
                movies["m"+m_id].genres.append(genre)
            genres.add(genre)

    return movies, directors, actors, genres 
Example 46
Project: chopsticks   Author: lordmauve   File: tunnel.py    (license) View Source Project 4 votes vote down vote up
def _connect_async(self, callback):
        if self.connected:
            callback(None)
            return
        try:
            path = sys._chopsticks_path[:]
        except AttributeError:
            path = []
        path.append(self.host)

        if len(path) > chopsticks.DEPTH_LIMIT:
            raise DepthLimitExceeded(
                'Depth limit of %s exceeded at %s' % (
                    chopsticks.DEPTH_LIMIT,
                    ' -> '.join(path)
                )
            )

        self.connect_pipes()
        self.reader = loop.reader(self.rpipe, self)
        self.writer = loop.writer(self.wpipe)

        def wrapped_callback(res):
            self.connected = not isinstance(res, ErrorResult)
            if self.connected:
                # Remote sends a pickle_version in response to OP_START
                self.pickle_version = min(self.HIGHEST_PICKLE_PROTOCOL, res)
            callback(res)
        self.callbacks[0] = wrapped_callback

        self.reader.start()
        self.writer.write_raw(bubble)

        self.write_msg(
            OP_START,
            req_id=0,
            host=self.host,
            path=path,
            depthlimit=chopsticks.DEPTH_LIMIT,
        )

        self.errreader = ioloop.StderrReader(errloop, self.epipe, self.host)
        start_errloop()