1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 __all__ = ["StorageLevel"]
19
20
22
23 """
24 Flags for controlling the storage of an RDD. Each StorageLevel records whether to use memory,
25 whether to drop the RDD to disk if it falls out of memory, whether to keep the data in memory
26 in a serialized format, and whether to replicate the RDD partitions on multiple nodes.
27 Also contains static constants for some commonly used storage levels, such as MEMORY_ONLY.
28 """
29
30 - def __init__(self, useDisk, useMemory, useOffHeap, deserialized, replication=1):
31 self.useDisk = useDisk
32 self.useMemory = useMemory
33 self.useOffHeap = useOffHeap
34 self.deserialized = deserialized
35 self.replication = replication
36
38 return "StorageLevel(%s, %s, %s, %s, %s)" % (
39 self.useDisk, self.useMemory, self.useOffHeap, self.deserialized, self.replication)
40
42 result = ""
43 result += "Disk " if self.useDisk else ""
44 result += "Memory " if self.useMemory else ""
45 result += "Tachyon " if self.useOffHeap else ""
46 result += "Deserialized " if self.deserialized else "Serialized "
47 result += "%sx Replicated" % self.replication
48 return result
49
50 StorageLevel.DISK_ONLY = StorageLevel(True, False, False, False)
51 StorageLevel.DISK_ONLY_2 = StorageLevel(True, False, False, False, 2)
52 StorageLevel.MEMORY_ONLY = StorageLevel(False, True, False, True)
53 StorageLevel.MEMORY_ONLY_2 = StorageLevel(False, True, False, True, 2)
54 StorageLevel.MEMORY_ONLY_SER = StorageLevel(False, True, False, False)
55 StorageLevel.MEMORY_ONLY_SER_2 = StorageLevel(False, True, False, False, 2)
56 StorageLevel.MEMORY_AND_DISK = StorageLevel(True, True, False, True)
57 StorageLevel.MEMORY_AND_DISK_2 = StorageLevel(True, True, False, True, 2)
58 StorageLevel.MEMORY_AND_DISK_SER = StorageLevel(True, True, False, False)
59 StorageLevel.MEMORY_AND_DISK_SER_2 = StorageLevel(True, True, False, False, 2)
60 StorageLevel.OFF_HEAP = StorageLevel(False, False, True, False, 1)
61