checking in some stuff... i'm going to rework how i do the disk init to ALWAYS use freshDisk().

This commit is contained in:
brent s 2019-10-22 14:34:39 -04:00
parent 108588827a
commit 305a0db34f
8 changed files with 227 additions and 33 deletions

View File

@ -48,10 +48,10 @@
</xs:simpleType>

<xs:simpleType name="fstype">
<!-- ', '.join(sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys()))) -->
<xs:annotation>
<xs:documentation>
This element validates a filesystem type to be specified for formatting a partition. See sgdisk -L (or
the table at http://www.rodsbooks.com/gdisk/walkthrough.html) for valid filesystem codes.
This element validates a filesystem type to be specified for formatting a partition (NOT applying a filesystem!); valid values are: affs0, affs1, affs2, affs3, affs4, affs5, affs6, affs7, amufs, amufs0, amufs1, amufs2, amufs3, amufs4, amufs5, apfs1, apfs2, asfs, btrfs, ext2, ext3, ext4, fat16, fat32, hfs, hfs+, hfsx, hp-ufs, jfs, linux-swap(v0), linux-swap(v1), nilfs2, ntfs, reiserfs, sun-ufs, swsusp, udf, xfs
</xs:documentation>
</xs:annotation>
<xs:restriction base="xs:token">

View File

@ -2,6 +2,8 @@ import os
##
from lxml import etree

#https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635 ?

class Config(object):
def __init__(self):
self.xml = None

View File

@ -0,0 +1,4 @@
import parted


PARTED_FSTYPES = list(dict(vars(parted.filesystem))['fileSystemType'].keys())

View File

@ -19,6 +19,7 @@ import parted # https://www.gnu.org/software/parted/api/index.html
import psutil
##
from .aif_util import xmlBool
from .constants import PARTED_FSTYPES


# parted lib can do SI or IEC (see table to right at https://en.wikipedia.org/wiki/Binary_prefix)
@ -60,55 +61,76 @@ def convertSizeUnit(pos):


class Partition(object):
def __init__(self, disk_xml, diskobj, start_sector):
self.xml = disk_xml
device = diskobj.device
def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype):
if tbltype not in ('gpt', 'msdos'):
raise ValueError('{0} must be one of gpt or msdos'.format(tbltype))
self.xml = part_xml
self.partnum = partnum
self.fstype = self.xml.attrib['fsType'].lower()
if self.fstype not in PARTED_FSTYPES:
raise ValueError(('{0} is not a valid partition filesystem type; '
'must be one of: {1}').format(self.xml.attrib['fsType'],
', '.join(sorted(PARTED_FSTYPES))))
self.disk = diskobj
self.device = self.disk.device
self.dev = '{0}{1}'.format(self.device.path, partnum)
sizes = {}
for s in ('start', 'stop'):
x = dict(zip(('from_bgn', 'size', 'type'),
convertSizeUnit(self.xml.attrib[s])))
sectors = x['size']
if x['type'] == '%':
sectors = int(device.getLength() / x['size'])
sectors = int(self.device.getLength() / x['size'])
elif x['type'] in _units.keys():
sectors = int(x['size'] << _units[x['type']] / device.sectorSize)
sectors = int(x['size'] << _units[x['type']] / self.device.sectorSize)
sizes[s] = (sectors, x['from_bgn'])
if sizes['start'][1] is not None:
if sizes['start'][1]:
self.begin = sizes['start'][0] + 0
else:
self.begin = device.getLength() - sizes['start'][0]
self.begin = self.device.getLength() - sizes['start'][0]
else:
self.begin = sizes['start'][0] + start_sector
if sizes['stop'][1] is not None:
if sizes['stop'][1]:
self.end = sizes['stop'][0] + 0
else:
self.end = device.getLength() - sizes['stop'][0]
# This *technically* should be - 34, but the alignment optimizer fixes it for us.
self.end = (self.device.getLength() - 1) - sizes['stop'][0]
else:
self.end = self.begin + sizes['stop'][0]
# TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit
# in configs. So we manually crunch the numbers and do it all at the end.
self.geometry = parted.Geometry(device = device,
self.geometry = parted.Geometry(device = self.device,
start = self.begin,
end = self.end)
self.filesystem = parted.FileSystem(type = self.xml.attrib['fsType'],
)
self.filesystem = parted.FileSystem(type = self.fstype,
geometry = self.geometry)
self.partition = parted.Partition(disk = diskobj,
type = parted.PARTITION_NORMAL,
geometry = self.geometry,
fs = )
fs = self.filesystem)
if tbltype == 'gpt' and self.xml.attrib.get('name'):
# The name attribute setting is b0rk3n, so we operate on the underlying PedPartition object.
# https://github.com/dcantrell/pyparted/issues/49#issuecomment-540096687
# https://github.com/dcantrell/pyparted/issues/65
# self.partition.name = self.xml.attrib.get('name')
_pedpart = self.partition.getPedPartition()
_pedpart.set_name(self.xml.attrib.get('name'))


class Disk(object):
def __init__(self, disk_xml):
self.xml = disk_xml
self.devpath = self.xml.attrib['device']
self.partitions = []
self._initDisk()

def _initDisk(self):
self.device = parted.getDevice(self.devpath)
try:
self.disk = parted.newDisk(self.device)
self.is_new = False
if xmlBool(self.xml.attrib.get('forceReformat')):
self.is_lowformatted = False
self.is_hiformatted = False
@ -121,43 +143,71 @@ class Disk(object):
if d.get('fstype', '').strip() != '':
self.is_hiformatted = True
break
self.is_partitioned = True
except parted._ped.DiskException:
self.disk = None
self.is_new = True
self.is_lowformatted = False
self.is_hiformatted = False
self.is_partitioned = False
self.is_partitioned = False
return()

def diskformat(self):
def diskFormat(self):
if self.is_lowformatted:
return()
# This is a safeguard. We do *not* want to low-format a disk that is mounted.
for p in psutil.disk_partitions(all = True):
if self.devpath in p:
raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath))
self.disk.deleteAllPartitions()
tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower()
if tabletype in ('bios', 'mbr'):
tabletype = 'msdos'
if not self.is_new:
self.disk.deleteAllPartitions()
self.tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower()
if self.tabletype in ('bios', 'mbr', 'dos'):
self.tabletype = 'msdos'
validlabels = parted.getLabels()
if tabletype not in validlabels:
if self.tabletype not in validlabels:
raise ValueError(('Disk format {0} is not valid for this architecture;'
'must be one of: {1}'.format(tabletype, ', '.join(list(validlabels)))))
self.disk = parted.freshDisk(self.device, tabletype)

pass
'must be one of: {1}'.format(self.tabletype, ', '.join(list(validlabels)))))
self.disk = parted.freshDisk(self.device, self.tabletype)
self.is_lowformatted = True
self.is_partitioned = True
return()

def fsformat(self):
def fsFormat(self):
if self.is_hiformatted:
return()
# This is a safeguard. We do *not* want to high-format a disk that is mounted.
for p in psutil.disk_partitions(all = True):
if self.devpath in p:
raise RuntimeError('{0} is mounted; we are cowardly refusing to high-format it'.format(self.devpath))

# TODO!
pass
return()

def getPartitions(self):
# For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment
# optimizer fixes it for us automatically.
start_sector = 0
self.partitions = []
for idx, part in enumerate(self.xml.findall('part')):
p = Partition(part, self.disk, start_sector, idx + 1, self.tabletype)
start_sector = p.end + 1
self.partitions.append(p)
return()

def partFormat(self):
if self.is_partitioned:
return()
if not self.is_lowformatted:
self.diskFormat()
# This is a safeguard. We do *not* want to partition a disk that is mounted.
for p in psutil.disk_partitions(all = True):
if self.devpath in p:
raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath))
if not self.partitions:
self.getPartitions()
if not self.partitions:
return()
for p in self.partitions:
self.disk.addPartition(partition = p, constraint = self.device.optimalAlignedConstraint)
self.disk.commit()
self.is_partitioned = True
return()

View File

@ -199,7 +199,7 @@ The `/aif` element is the https://en.wikipedia.org/wiki/Root_element[root elemen
The `/aif/storage` element contains <<code_disk_code, disk>>, <<code_part_code, disk/part>>, and <<code_mount_code, mount>> elements.

==== `<disk>`
The `/aif/storage/disk` element holds information about disks on the system, and within this element are one (or more) <<code_part_code, part>> elements.
The `/aif/storage/disk` element holds information about disks on the system, and within this element are one (or more) <<code_part_code, part>> elements. Note that any `disk` elements specified here will be *entirely reformatted*; operate under the assumption that ANY and ALL pre-existing data on the specified device will be IRREVOCABLY LOST.

[options="header"]
|======================
@ -544,7 +544,19 @@ There are several script types availabe for `execution`. Currently, these are:
*pre* scripts are run (in numerical `order`) before the disks are even formatted. *pkg* scripts are run (in numerical `order`) right before the <<code_package_code, packages>> are installed (this allows you to configure an <<command, alternate packager>> such as https://aur.archlinux.org/packages/apacman/[apacman^]) -- these are run *inside* the chroot of the new install. *post* scripts are run inside the chroot like *pkg*, but are executed very last thing, just before the reboot.

= Further Information
Here you will find further info, other resources, and such relating to AIF-NG.
Here you will find further info and other resources relating to AIF-NG.

== FAQ

=== "I specified start sector as 0 for a GPT-labeled disk but it starts at sector 2048 instead. What gives?"
GPT requires 33 sectors for the table at the beginning (and 32 sectors at the end) for the actual table. That plus an extra (usually) 512 bytes at the beginning for something called a https://en.wikipedia.org/wiki/GUID_Partition_Table#Protective_MBR_(LBA_0)[Protective MBR^] (this prevents disk utilities from overwriting the GPT label automatically in case they only recognize "msdos" labels and assume the disk is not formatted yet).

Most disks these days use something called https://en.wikipedia.org/wiki/Advanced_Format[Advanced Format^]. These align their sectors to factors of 8, so sector 34 can't be used - it'd have to be sector 40. Additionally, various other low-level disk interactions (e.g. RAID stripe sizes) require a much larger boundary between partitions. If you're interested in a little more detail, you may find https://metebalci.com/blog/a-quick-tour-of-guid-partition-table-gpt/[this^] interesting (specifically https://metebalci.com/blog/a-quick-tour-of-guid-partition-table-gpt/#gpt-partition-entry-array[this section^], paragraph starting with `You may also ask why the first partition starts from LBA 2048...`).

TL;DR: "It's the safest way to make sure your disk doesn't suffer massive degradation in performance, your RAID doesn't eat partitions, etc." Don't worry, it typically only ends up being about 1MB of "wasted" space surrounding partitions. I've written plaintext documentation larger than 1MB.

=== "Why isn't my last GPT partition extending to the last sector?"
See above.

== Bug Reports/Feature Requests
NOTE: It is possible to submit a bug or feature request without registering in my bugtracker. One of my pet peeves is needing to create an account/register on a bugtracker simply to report a bug! The following links only require an email address to file a bug (which is necessary in case I need any further clarification from you or to keep you updated on the status of the bug/feature request -- so please be sure to use a valid email address).

View File

@ -46,3 +46,6 @@ https://www.w3schools.com/xml/schema_intro.asp
https://www.w3schools.com/xml/schema_example.asp
https://msdn.microsoft.com/en-us/library/dd489258.aspx

if i ever need a list of GPT GUIDs, maybe to do some fancy GUID-to-name-and-back mapping?
https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs
(mapping can be done via https://stackoverflow.com/questions/483666/reverse-invert-a-dictionary-mapping)

View File

@ -5,13 +5,18 @@
<storage>
<disk device="/dev/sda" diskFormat="gpt" forceReformat="true">
<!-- Partitions are numbered *in the order they are specified*. -->
<part id="boot" label="/boot" start="0%" stop="10%" fsType="ef00" /><!-- e.g. this would be /dev/sda1 -->
<part id="secrets1" label="shh" start="10%" stop="20%" fsType="8300"/>
<part id="lvm_member1" label="dynamic" start="20%" stop="30%" fsType="8300"/>
<part id="boot" name="BOOT" label="/boot" start="0%" stop="10%" fsType="ef00"/><!-- e.g. this would be /dev/sda1 -->
<part id="secrets1" name="crypted" label="shh" start="10%" stop="20%" fsType="8300"/>
<part id="lvm_member1" name="jbod" label="dynamic" start="20%" stop="30%" fsType="8300"/>
<part id="raid1_d1" start="30%" stop="55%" fsType="fd00"/>
<part id="raid1_d2" start="55%" stop="80%" fsType="fd00"/>
<part id="swap" start="80%" stop="100%" fsType="8200" />
</disk>
<fileSystems>
<fs source="boot" type="vfat">
<opts>-F 32 -n {label}</opts>
</fs>
</fileSystems>
<!-- "Special" devices are processed *in the order they are specified*. This is important if you wish to
e.g. layer LUKS on top of LVM - you would specify <lvm> before <luks> and reference the
<luksDev id="SOMETHING" ... > as <lvmLogical source="SOMETHING" ... />. -->

118
docs/reference/fstypes Normal file
View File

@ -0,0 +1,118 @@
msdos:
0 Empty 24 NEC DOS 81 Minix / old Lin bf Solaris
1 FAT12 27 Hidden NTFS Win 82 Linux swap / So c1 DRDOS/sec (FAT-
2 XENIX root 39 Plan 9 83 Linux c4 DRDOS/sec (FAT-
3 XENIX usr 3c PartitionMagic 84 OS/2 hidden or c6 DRDOS/sec (FAT-
4 FAT16 <32M 40 Venix 80286 85 Linux extended c7 Syrinx
5 Extended 41 PPC PReP Boot 86 NTFS volume set da Non-FS data
6 FAT16 42 SFS 87 NTFS volume set db CP/M / CTOS / .
7 HPFS/NTFS/exFAT 4d QNX4.x 88 Linux plaintext de Dell Utility
8 AIX 4e QNX4.x 2nd part 8e Linux LVM df BootIt
9 AIX bootable 4f QNX4.x 3rd part 93 Amoeba e1 DOS access
a OS/2 Boot Manag 50 OnTrack DM 94 Amoeba BBT e3 DOS R/O
b W95 FAT32 51 OnTrack DM6 Aux 9f BSD/OS e4 SpeedStor
c W95 FAT32 (LBA) 52 CP/M a0 IBM Thinkpad hi ea Rufus alignment
e W95 FAT16 (LBA) 53 OnTrack DM6 Aux a5 FreeBSD eb BeOS fs
f W95 Ext'd (LBA) 54 OnTrackDM6 a6 OpenBSD ee GPT
10 OPUS 55 EZ-Drive a7 NeXTSTEP ef EFI (FAT-12/16/
11 Hidden FAT12 56 Golden Bow a8 Darwin UFS f0 Linux/PA-RISC b
12 Compaq diagnost 5c Priam Edisk a9 NetBSD f1 SpeedStor
14 Hidden FAT16 <3 61 SpeedStor ab Darwin boot f4 SpeedStor
16 Hidden FAT16 63 GNU HURD or Sys af HFS / HFS+ f2 DOS secondary
17 Hidden HPFS/NTF 64 Novell Netware b7 BSDI fs fb VMware VMFS
18 AST SmartSleep 65 Novell Netware b8 BSDI swap fc VMware VMKCORE
1b Hidden W95 FAT3 70 DiskSecure Mult bb Boot Wizard hid fd Linux raid auto
1c Hidden W95 FAT3 75 PC/IX bc Acronis FAT32 L fe LANstep
1e Hidden W95 FAT1 80 Old Minix be Solaris boot ff BBT

gpt:
1 EFI System C12A7328-F81F-11D2-BA4B-00A0C93EC93B
2 MBR partition scheme 024DEE41-33E7-11D3-9D69-0008C781F39F
3 Intel Fast Flash D3BFE2DE-3DAF-11DF-BA40-E3A556D89593
4 BIOS boot 21686148-6449-6E6F-744E-656564454649
5 Sony boot partition F4019732-066E-4E12-8273-346C5641494F
6 Lenovo boot partition BFBFAFE7-A34F-448A-9A5B-6213EB736C22
7 PowerPC PReP boot 9E1A2D38-C612-4316-AA26-8B49521E5A8B
8 ONIE boot 7412F7D5-A156-4B13-81DC-867174929325
9 ONIE config D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149
10 Microsoft reserved E3C9E316-0B5C-4DB8-817D-F92DF00215AE
11 Microsoft basic data EBD0A0A2-B9E5-4433-87C0-68B6B72699C7
12 Microsoft LDM metadata 5808C8AA-7E8F-42E0-85D2-E1E90434CFB3
13 Microsoft LDM data AF9B60A0-1431-4F62-BC68-3311714A69AD
14 Windows recovery environment DE94BBA4-06D1-4D40-A16A-BFD50179D6AC
15 IBM General Parallel Fs 37AFFC90-EF7D-4E96-91C3-2D7AE055B174
16 Microsoft Storage Spaces E75CAF8F-F680-4CEE-AFA3-B001E56EFC2D
17 HP-UX data 75894C1E-3AEB-11D3-B7C1-7B03A0000000
18 HP-UX service E2A1E728-32E3-11D6-A682-7B03A0000000
19 Linux swap 0657FD6D-A4AB-43C4-84E5-0933C84B4F4F
20 Linux filesystem 0FC63DAF-8483-4772-8E79-3D69D8477DE4
21 Linux server data 3B8F8425-20E0-4F3B-907F-1A25A76F98E8
22 Linux root (x86) 44479540-F297-41B2-9AF7-D131D5F0458A
23 Linux root (ARM) 69DAD710-2CE4-4E3C-B16C-21A1D49ABED3
24 Linux root (x86-64) 4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709
25 Linux root (ARM-64) B921B045-1DF0-41C3-AF44-4C6F280D3FAE
26 Linux root (IA-64) 993D8D3D-F80E-4225-855A-9DAF8ED7EA97
27 Linux reserved 8DA63339-0007-60C0-C436-083AC8230908
28 Linux home 933AC7E1-2EB4-4F13-B844-0E14E2AEF915
29 Linux RAID A19D880F-05FC-4D3B-A006-743F0F84911E
30 Linux extended boot BC13C2FF-59E6-4262-A352-B275FD6F7172
31 Linux LVM E6D6D379-F507-44C2-A23C-238F2A3DF928
32 FreeBSD data 516E7CB4-6ECF-11D6-8FF8-00022D09712B
33 FreeBSD boot 83BD6B9D-7F41-11DC-BE0B-001560B84F0F
34 FreeBSD swap 516E7CB5-6ECF-11D6-8FF8-00022D09712B
35 FreeBSD UFS 516E7CB6-6ECF-11D6-8FF8-00022D09712B
36 FreeBSD ZFS 516E7CBA-6ECF-11D6-8FF8-00022D09712B
37 FreeBSD Vinum 516E7CB8-6ECF-11D6-8FF8-00022D09712B
38 Apple HFS/HFS+ 48465300-0000-11AA-AA11-00306543ECAC
39 Apple UFS 55465300-0000-11AA-AA11-00306543ECAC
40 Apple RAID 52414944-0000-11AA-AA11-00306543ECAC
41 Apple RAID offline 52414944-5F4F-11AA-AA11-00306543ECAC
42 Apple boot 426F6F74-0000-11AA-AA11-00306543ECAC
43 Apple label 4C616265-6C00-11AA-AA11-00306543ECAC
44 Apple TV recovery 5265636F-7665-11AA-AA11-00306543ECAC
45 Apple Core storage 53746F72-6167-11AA-AA11-00306543ECAC
46 Solaris boot 6A82CB45-1DD2-11B2-99A6-080020736631
47 Solaris root 6A85CF4D-1DD2-11B2-99A6-080020736631
48 Solaris /usr & Apple ZFS 6A898CC3-1DD2-11B2-99A6-080020736631
49 Solaris swap 6A87C46F-1DD2-11B2-99A6-080020736631
50 Solaris backup 6A8B642B-1DD2-11B2-99A6-080020736631
51 Solaris /var 6A8EF2E9-1DD2-11B2-99A6-080020736631
52 Solaris /home 6A90BA39-1DD2-11B2-99A6-080020736631
53 Solaris alternate sector 6A9283A5-1DD2-11B2-99A6-080020736631
54 Solaris reserved 1 6A945A3B-1DD2-11B2-99A6-080020736631
55 Solaris reserved 2 6A9630D1-1DD2-11B2-99A6-080020736631
56 Solaris reserved 3 6A980767-1DD2-11B2-99A6-080020736631
57 Solaris reserved 4 6A96237F-1DD2-11B2-99A6-080020736631
58 Solaris reserved 5 6A8D2AC7-1DD2-11B2-99A6-080020736631
59 NetBSD swap 49F48D32-B10E-11DC-B99B-0019D1879648
60 NetBSD FFS 49F48D5A-B10E-11DC-B99B-0019D1879648
61 NetBSD LFS 49F48D82-B10E-11DC-B99B-0019D1879648
62 NetBSD concatenated 2DB519C4-B10E-11DC-B99B-0019D1879648
63 NetBSD encrypted 2DB519EC-B10E-11DC-B99B-0019D1879648
64 NetBSD RAID 49F48DAA-B10E-11DC-B99B-0019D1879648
65 ChromeOS kernel FE3A2A5D-4F32-41A7-B725-ACCC3285A309
66 ChromeOS root fs 3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC
67 ChromeOS reserved 2E0A753D-9E48-43B0-8337-B15192CB1B5E
68 MidnightBSD data 85D5E45A-237C-11E1-B4B3-E89A8F7FC3A7
69 MidnightBSD boot 85D5E45E-237C-11E1-B4B3-E89A8F7FC3A7
70 MidnightBSD swap 85D5E45B-237C-11E1-B4B3-E89A8F7FC3A7
71 MidnightBSD UFS 0394EF8B-237E-11E1-B4B3-E89A8F7FC3A7
72 MidnightBSD ZFS 85D5E45D-237C-11E1-B4B3-E89A8F7FC3A7
73 MidnightBSD Vinum 85D5E45C-237C-11E1-B4B3-E89A8F7FC3A7
74 Ceph Journal 45B0969E-9B03-4F30-B4C6-B4B80CEFF106
75 Ceph Encrypted Journal 45B0969E-9B03-4F30-B4C6-5EC00CEFF106
76 Ceph OSD 4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D
77 Ceph crypt OSD 4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D
78 Ceph disk in creation 89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE
79 Ceph crypt disk in creation 89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE
80 VMware VMFS AA31E02A-400F-11DB-9590-000C2911D1B8
81 VMware Diagnostic 9D275380-40AD-11DB-BF97-000C2911D1B8
82 VMware Virtual SAN 381CFCCC-7288-11E0-92EE-000C2911D0B2
83 VMware Virsto 77719A0C-A4A0-11E3-A47E-000C29745A24
84 VMware Reserved 9198EFFC-31C0-11DB-8F78-000C2911D1B8
85 OpenBSD data 824CC7A0-36A8-11E3-890A-952519AD3F61
86 QNX6 file system CEF5A9AD-73BC-4601-89F3-CDEEEEE321A1
87 Plan 9 partition C91818F9-8025-47AF-89D2-F030D7000C2C
88 HiFive Unleashed FSBL 5B193300-FC78-40CD-8002-E86C45580B47
89 HiFive Unleashed BBL 2E54B353-1271-4842-806F-E436D6AF6985