Add Makefile

Makefile

@@ -0,0 +1,12 @@
+build:
+	py src/bulkloading.py data/offsets.txt data/coords.txt output/rtree.txt
+
+range:
+	py src/rangequeries.py output/rtree.txt data/ranges.txt
+
+knn:
+	py src/knnqueries.py output/rtree.txt data/knn.txt 10
+
+.PHONY: clean
+clean:
+	rm -rf output

data/knn.txt

@@ -0,0 +1,100 @@
+-86.7883 32.3534
+-86.6036 32.4001
+-86.7958 32.3467
+-86.5878 32.3816
+-86.7666 32.469
+-86.6607 32.4106
+-86.6078 32.4385
+-86.6154 32.5194
+-86.7997 32.4033
+-86.5795 32.458
+-86.7369 32.4667
+-86.747 32.5155
+-86.7435 32.4913
+-86.6512 32.4148
+-86.6413 32.4371
+-86.7377 32.4732
+-86.8318 32.4539
+-86.6761 32.3988
+-86.7472 32.4673
+-86.9008 32.5384
+-86.9 32.5352
+-86.8986 32.5348
+-86.6946 32.4956
+-86.9034 32.5839
+-86.8993 32.5491
+-86.8705 32.5674
+-86.7056 32.6863
+-86.714 32.7018
+-86.652 32.5832
+-86.6402 32.7001
+-86.6941 32.6697
+-86.9135 32.6635
+-86.9157 32.6606
+-86.9172 32.6601
+-86.9157 32.6571
+-86.9163 32.6586
+-86.9153 32.6422
+-86.8982 32.5717
+-86.8964 32.5711
+-86.8952 32.5615
+-86.8953 32.5601
+-86.8931 32.5587
+-86.8985 32.5516
+-86.4891 32.6747
+-86.4566 32.694
+-86.5442 32.6503
+-86.4964 32.7051
+-86.4724 32.6947
+-86.5161 32.6741
+-86.4484 32.666
+-86.4955 32.6661
+-86.4318 32.6047
+-86.6084 32.6348
+-86.4408 32.7067
+-86.4696 32.6969
+-86.5684 32.5844
+-86.5698 32.5839
+-86.49 32.6172
+-86.4606 32.6986
+-86.489 32.5946
+-86.5054 32.5992
+-86.5034 32.5311
+-86.5017 32.3956
+-86.5442 32.3911
+-86.5424 32.3874
+-86.5416 32.5043
+-86.4465 32.5841
+-86.5697 32.3965
+-86.5155 32.5496
+-86.561 32.4232
+-86.5131 32.5492
+-86.5169 32.4217
+-86.531 32.3859
+-86.5495 32.5033
+-86.5261 32.4445
+-86.4447 32.4158
+-86.4162 32.4778
+-86.5138 32.5314
+-86.465 32.4491
+-86.465 32.4488
+-86.4216 32.4226
+-86.5703 32.5028
+-86.4907 32.4468
+-86.4653 32.445
+-86.4231 32.4322
+-86.6809 32.3598
+-86.8579 32.3212
+-86.5931 32.3688
+-86.8326 32.439
+-86.9212 32.4375
+-86.6843 32.3892
+-86.615 32.5147
+-86.7969 32.3962
+-86.8034 32.3907
+-86.7942 32.3569
+-86.7879 32.4106
+-86.67 32.4255
+-86.7852 32.3339
+-86.8026 32.5166
+-86.7 32.4229

src/bulkloading.py

@@ -0,0 +1,143 @@
+from os import write
+from os import path, makedirs
+import sys
+import csv
+
+if not path.exists('output'):
+    makedirs('output')
+
+_DIVISORS = [180.0 / 2 ** n for n in range(32)]
+
+class MBR:
+    def __init__(self, id, xlow, xhigh, ylow, yhigh):
+        self.xlow = xlow
+        self.xhigh = xhigh
+        self.ylow = ylow
+        self.yhigh = yhigh
+        self.id = id
+        self.zcurve = self.findZcurve()
+
+    def findZcurve(self):
+        x_median = (self.xlow + self.xhigh) / 2
+        y_median = (self.ylow + self.yhigh) / 2
+        return interleave_latlng(y_median, x_median)
+
+def interleave_latlng(lat, lng):
+    if lng > 180:
+        x = (lng % 180) + 180.0
+    elif lng < -180:
+        x = (-((-lng) % 180)) + 180.0
+    else:
+        x = lng + 180.0
+    if lat > 90:
+        y = (lat % 90) + 90.0
+    elif lat < -90:
+        y = (-((-lat) % 90)) + 90.0
+    else:
+        y = lat + 90.0
+
+    morton_code = ""
+    for dx in _DIVISORS:
+        digit = 0
+        if y >= dx:
+            digit |= 2
+            y -= dx
+        if x >= dx:
+            digit |= 1
+            x -= dx
+        morton_code += str(digit)
+
+    return morton_code
+
+# Given a set of coordinates find the MBR
+def findMBR(objId, points):
+    x_min = min(points, key=lambda p: p[0])[0]
+    x_max = max(points, key=lambda p: p[0])[0]
+    y_min = min(points, key=lambda p: p[1])[1]
+    y_max = max(points, key=lambda p: p[1])[1]
+
+    return MBR(objId, x_min, x_max, y_min, y_max)
+
+# Given a set of MBRs find the MBR
+def createMBR(nodeId, mbrs):
+    x_min = min(mbr.xlow for mbr in mbrs)
+    x_max = max(mbr.xhigh for mbr in mbrs)
+    y_min = min(mbr.ylow for mbr in mbrs)
+    y_max = max(mbr.yhigh for mbr in mbrs)
+
+    return MBR(nodeId, x_min, x_max, y_min, y_max)
+
+# Read data from input files
+def inputReader(filename1, filename2):
+    set_of_points = []
+    with open(filename1, 'r') as offsets, open(filename2, 'r') as coords:
+        reader = csv.reader(offsets, delimiter=',', quoting=csv.QUOTE_NONE)
+        for row in reader:
+            points = []
+            for i in range(int(row[2]) - int(row[1]) + 1):
+                points.append([float(x) for x in coords.readline().split(',')])
+            set_of_points.append([row[0], points])
+
+    return set_of_points
+
+# Write data into output file
+def outputWriter(filename, rTree):
+    with open(filename, 'w+') as rtree:
+        numNode = 0
+        leaves = 0
+        for level in rTree:
+            for node in level:
+                rtree.write("[{}, {}, [[".format(leaves, numNode))
+                for i in range(len(node) - 1):
+                    rtree.write("{}, [{}, {}, {}, {}]],[".format(
+                        node[i].id, node[i].xlow, node[i].xhigh, node[i].ylow, node[i].yhigh))
+                rtree.write("{}, [{}, {}, {}, {}]]]]".format(
+                        node[-1].id, node[-1].xlow, node[-1].xhigh, node[-1].ylow, node[-1].yhigh))
+                rtree.write("\n")
+                numNode += 1
+            leaves = 1
+
+#
+def makeRtree(collection):
+    level = [] # list of levels(of nodes) of nodes(of mbrs) of mbrs(of points)
+    nodeId = 0
+    rank = 0
+    # Until we reach the root node
+    while len(collection) > 1:
+        # Split the mbr collection into nodes of 20
+        nodes = [collection[x:x+20] for x in range(0, len(collection), 20)]
+
+        # If the last node has less than 8 mbrs, fill with mbrs of the previous
+        balance = 8 - len(nodes[-1])
+        if balance > 0 and len(nodes) > 1:
+            migrate = len(nodes[-2])
+            nodes[-1] = nodes[-2][migrate-balance:] + nodes[-1]
+            nodes[-2] = nodes[-2][:migrate-balance]
+
+        # Make the new MBRs based on the corners of each 20-piece
+        collection = []
+        for node in nodes: #for every 20 MBRs in 500 MBRs
+            collection.append(createMBR(nodeId, node))
+            nodeId += 1
+
+        level.append(nodes)
+        print("{} nodes at level {}".format(len(nodes), rank))
+        rank += 1
+
+    return level
+
+if __name__ == '__main__':
+    # Read the coordinates of the polygons points
+    set_of_points = inputReader(filename1=sys.argv[1], filename2=sys.argv[2])
+
+    # Create the mbrs of the polygons
+    mbrs = []
+    for sp in set_of_points:
+        mbrs.append(findMBR(sp[0], sp[1]))
+
+    # Sort the mbrs based on the Z curve
+    mbrs.sort(key=lambda ld: ld.zcurve)
+
+    # Make the Rtree
+    rTree = makeRtree(mbrs)
+    outputWriter("output/rtree.txt", rTree)

src/knnqueries.py

@@ -0,0 +1,116 @@
+import sys
+import ast
+import heapq as hq
+from math import sqrt
+
+class Point:
+	def __init__(self, x, y):
+		self.x = x
+		self.y = y
+
+class MBR:
+    def __init__(self, id, xlow, xhigh, ylow, yhigh):
+        self.xlow = xlow
+        self.xhigh = xhigh
+        self.ylow = ylow
+        self.yhigh = yhigh
+        self.id = id
+        self.obj = False
+
+    def setDistance(self, qp):
+        self.distance = self.findDistance(qp)
+
+    def findDistance(self, qp):
+        dx = dy = 0
+        if qp.x < self.xlow:
+            dx = self.xlow - qp.x
+        elif qp.x > self.xhigh:
+            dx = qp.x - self.xhigh
+        else:
+            dx = 0
+
+        if qp.y < self.ylow:
+            dy = self.ylow - qp.y
+        elif qp.y > self.yhigh:
+            dy = qp.y - self.yhigh
+        else:
+            dy = 0
+        return sqrt(dx**2 + dy**2)
+
+    def makeObject(self):
+        self.obj = True
+
+    def __lt__(self, other):
+        if self.distance < other.distance:
+            return True
+        return False
+
+class Node:
+    def __init__(self, leaf, id, list_of_mbrs):
+        self.leaf = leaf
+        self.id = id
+        self.list_of_mbrs = list_of_mbrs
+
+def parseRtree(filename):
+    with open(filename, 'r') as f:
+        nodes = []
+        for line in f:
+            node = ast.literal_eval(line)
+            mbrs = []
+            for mbr in node[2]:
+                mbrs.append(MBR(int(mbr[0]), float(mbr[1][0]), float(mbr[1][1]),
+                    float(mbr[1][2]), float(mbr[1][3])))
+            nodes.append(Node(node[0], node[1], mbrs))
+
+    return nodes
+
+# Read and execute all the NN queries assigned to the r tree
+def parseQuery(rtree, filename, k):
+    with open(filename, 'r') as f:
+        num = 0
+        for line in f:
+            [x, y] = line.split()
+            findKNN(rtree, Point(float(x), float(y)), k, num)
+            num += 1
+
+# Search for the neareset k neighboors
+def findKNN(rtree, qp, k, num):
+    # List to put our k nn objects
+    results = []
+    # Initialize a priority queue
+    pq = []
+    # Put the root nodes into the priority queue
+    for mbr in rtree[-1].list_of_mbrs:
+        mbr.setDistance(qp)
+        hq.heappush(pq, mbr)
+
+    while len(pq) > 0 and len(results) < k:
+        # Retrieve the next node and remove it from the queue
+        e = hq.heappop(pq)
+
+        if e.obj == True:
+            hq.heappush(results, e)
+            continue
+
+        # At a leaf node
+        if rtree[e.id].leaf == 0:
+            for mbr in rtree[e.id].list_of_mbrs:
+                mbr.setDistance(qp)
+                mbr.makeObject()
+                hq.heappush(pq, mbr)
+        # At an intermediate node
+        else:
+            for node in rtree[e.id].list_of_mbrs:
+                node.setDistance(qp)
+                hq.heappush(pq, node)
+
+    print("{}:".format(num), end=' ')
+    for r in range(k-1):
+        nn = hq.heappop(results)
+        print("{},".format(nn.id), end='')
+    nn = hq.heappop(results)
+    print(nn.id)
+
+if __name__ == '__main__':
+    rtree = parseRtree(sys.argv[1])
+    parseQuery(rtree, sys.argv[2], int(sys.argv[3]))

src/rangequeries.py

@@ -0,0 +1,78 @@
+import sys
+import ast
+
+class MBR:
+    def __init__(self, id, xlow, xhigh, ylow, yhigh):
+        self.xlow = xlow
+        self.xhigh = xhigh
+        self.ylow = ylow
+        self.yhigh = yhigh
+        self.id = id
+
+    def intersects(self, other):
+        if self.xlow > other.xhigh or self.xhigh < other.xlow:
+            return False
+        if self.ylow > other.yhigh or self.yhigh < other.ylow:
+            return False
+        return True
+
+class Node:
+    def __init__(self, leaf, id, list_of_mbrs):
+        self.leaf = leaf
+        self.id = id
+        self.list_of_mbrs = list_of_mbrs
+
+def parseRtree(filename):
+    with open(filename, 'r') as f:
+        nodes = []
+        for line in f:
+            node = ast.literal_eval(line)
+            mbrs = []
+            for mbr in node[2]:
+                mbrs.append(MBR(int(mbr[0]), float(mbr[1][0]), float(mbr[1][1]),
+                    float(mbr[1][2]), float(mbr[1][3])))
+            nodes.append(Node(node[0], node[1], mbrs))
+
+    return nodes
+
+# Read and execute all the range queries assigned to the r tree
+def parseQuery(rtree, filename):
+    with open(filename, 'r') as f:
+        i = 0
+        for line in f:
+            [xlow, ylow, xhigh, yhigh] = line.split()
+            results = []
+            rangeQuery(rtree, MBR(int(i), float(xlow), float(xhigh),
+                    float(ylow), float(yhigh)), results)
+
+            print("{} ({}):".format(i, len(results)), end=' ')
+            for r in range(len(results) - 1):
+                print("{},".format(results[r].id), end='')
+            if len(results) != 0:
+                print(results[-1].id)
+            else:
+                print(' ')
+            i += 1
+
+# Find the intersecting rectangles
+def rangeQuery(rtree, window, results, node=None):
+    # At the root node
+    if node == None:
+        node = rtree[-1]
+        for n in node.list_of_mbrs:
+            if window.intersects(n) or n.intersects(window):
+                rangeQuery(rtree, window, results, rtree[n.id])
+    # At an intermediate node
+    elif node.leaf == 1:
+        for n in node.list_of_mbrs:
+            if window.intersects(n) or n.intersects(window):
+                rangeQuery(rtree, window, results, rtree[n.id])
+    # At a leaf node
+    else:
+        for mbr in node.list_of_mbrs:
+            if window.intersects(mbr) or mbr.intersects(window):
+                results.append(mbr)
+
+if __name__ == '__main__':
+    rtree = parseRtree(sys.argv[1])
+    parseQuery(rtree, sys.argv[2])