ДД
добавил в гуглодоку. Tested in и Platform - что написать?
SATA3 HBA или SATA3 Chipset? А платформа?
SATA3 HBA или SATA3 Chipset? А платформа?
Этого хватит? Или ещё что то нужно?
Size: a a a
2020 June 27
2020 June 28
ВН
Дмитрий Дятлов
Этого хватит? Или ещё что то нужно?
О..можешь еще 1 тест прогнать? То же что journal write с fsync, но -rw=randwrite, а не write
ДД
Это?
ВН
Дмитрий Дятлов
Это?
ага, вот так короче
fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdXXДД
ага, вот так короче
fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdXX[root@####### ~]# hdparm -W 0 /dev/sdb
/dev/sdb:
setting drive write-caching to 0 (off)
write-caching = 0 (off)
[root@####### ~]# fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdb
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.1
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=58.7MiB/s][r=0,w=15.0k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=811796: Sun Jun 28 00:16:19 2020
write: IOPS=15.7k, BW=61.5MiB/s (64.5MB/s)(3688MiB/60001msec)
slat (usec): min=2, max=7834, avg= 4.05, stdev= 9.27
clat (nsec): min=420, max=4612.9k, avg=57104.95, stdev=50852.58
lat (usec): min=28, max=7881, avg=61.23, stdev=51.80
clat percentiles (usec):
| 1.00th=[ 28], 5.00th=[ 28], 10.00th=[ 29], 20.00th=[ 30],
| 30.00th=[ 32], 40.00th=[ 34], 50.00th=[ 37], 60.00th=[ 42],
| 70.00th=[ 60], 80.00th=[ 80], 90.00th=[ 111], 95.00th=[ 145],
| 99.00th=[ 249], 99.50th=[ 273], 99.90th=[ 322], 99.95th=[ 367],
| 99.99th=[ 1352]
bw ( KiB/s): min=51696, max=94452, per=100.00%, avg=62981.71, stdev=5406.16, samples=119
iops : min=12924, max=23613, avg=15745.38, stdev=1351.56, samples=119
lat (nsec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=65.07%
lat (usec) : 100=22.42%, 250=11.49%, 500=0.92%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%
cpu : usr=3.30%, sys=11.56%, ctx=944136, majf=0, minf=58173
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwt: total=0,944210,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=61.5MiB/s (64.5MB/s), 61.5MiB/s-61.5MiB/s (64.5MB/s-64.5MB/s), io=3688MiB (3867MB), run=60001-60001msec
Disk stats (read/write):
sdb: ios=94/941200, merge=0/0, ticks=60/54156, in_queue=54216, util=99.69%
[root@####### ~]# hdparm -W 1 /dev/sdb
/dev/sdb:
setting drive write-caching to 0 (off)
write-caching = 0 (off)
[root@####### ~]# fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdb
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.1
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=58.7MiB/s][r=0,w=15.0k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=811796: Sun Jun 28 00:16:19 2020
write: IOPS=15.7k, BW=61.5MiB/s (64.5MB/s)(3688MiB/60001msec)
slat (usec): min=2, max=7834, avg= 4.05, stdev= 9.27
clat (nsec): min=420, max=4612.9k, avg=57104.95, stdev=50852.58
lat (usec): min=28, max=7881, avg=61.23, stdev=51.80
clat percentiles (usec):
| 1.00th=[ 28], 5.00th=[ 28], 10.00th=[ 29], 20.00th=[ 30],
| 30.00th=[ 32], 40.00th=[ 34], 50.00th=[ 37], 60.00th=[ 42],
| 70.00th=[ 60], 80.00th=[ 80], 90.00th=[ 111], 95.00th=[ 145],
| 99.00th=[ 249], 99.50th=[ 273], 99.90th=[ 322], 99.95th=[ 367],
| 99.99th=[ 1352]
bw ( KiB/s): min=51696, max=94452, per=100.00%, avg=62981.71, stdev=5406.16, samples=119
iops : min=12924, max=23613, avg=15745.38, stdev=1351.56, samples=119
lat (nsec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=65.07%
lat (usec) : 100=22.42%, 250=11.49%, 500=0.92%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%
cpu : usr=3.30%, sys=11.56%, ctx=944136, majf=0, minf=58173
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwt: total=0,944210,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=61.5MiB/s (64.5MB/s), 61.5MiB/s-61.5MiB/s (64.5MB/s-64.5MB/s), io=3688MiB (3867MB), run=60001-60001msec
Disk stats (read/write):
sdb: ios=94/941200, merge=0/0, ticks=60/54156, in_queue=54216, util=99.69%
[root@####### ~]# hdparm -W 1 /dev/sdb
ДД
ага, вот так короче
fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdXX/dev/sdb:
setting drive write-caching to 1 (on)
write-caching = 1 (on)
[root@####### ~]# fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdb
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.1
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=47.3MiB/s][r=0,w=12.1k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=812725: Sun Jun 28 00:17:31 2020
write: IOPS=11.8k, BW=46.1MiB/s (48.4MB/s)(2768MiB/60001msec)
slat (usec): min=2, max=3537, avg= 3.09, stdev= 4.94
clat (nsec): min=450, max=3856.2k, avg=34526.94, stdev=32327.92
lat (usec): min=25, max=3862, avg=37.70, stdev=32.74
clat percentiles (usec):
| 1.00th=[ 26], 5.00th=[ 27], 10.00th=[ 27], 20.00th=[ 29],
| 30.00th=[ 29], 40.00th=[ 30], 50.00th=[ 31], 60.00th=[ 33],
| 70.00th=[ 35], 80.00th=[ 37], 90.00th=[ 39], 95.00th=[ 43],
| 99.00th=[ 118], 99.50th=[ 163], 99.90th=[ 243], 99.95th=[ 537],
| 99.99th=[ 1467]
bw ( KiB/s): min=43302, max=50888, per=100.00%, avg=47237.77, stdev=1569.59, samples=119
iops : min=10825, max=12722, avg=11809.37, stdev=392.40, samples=119
lat (nsec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=96.47%
lat (usec) : 100=2.19%, 250=1.20%, 500=0.04%, 750=0.02%, 1000=0.02%
lat (msec) : 2=0.02%, 4=0.01%
cpu : usr=1.39%, sys=13.28%, ctx=2104360, majf=0, minf=59077
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwt: total=0,708611,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=46.1MiB/s (48.4MB/s), 46.1MiB/s-46.1MiB/s (48.4MB/s-48.4MB/s), io=2768MiB (2902MB), run=60001-60001msec
Disk stats (read/write):
sdb: ios=15/1412648, merge=0/0, ticks=5/54262, in_queue=83332, util=99.71%
setting drive write-caching to 1 (on)
write-caching = 1 (on)
[root@####### ~]# fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdb
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.1
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=47.3MiB/s][r=0,w=12.1k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=812725: Sun Jun 28 00:17:31 2020
write: IOPS=11.8k, BW=46.1MiB/s (48.4MB/s)(2768MiB/60001msec)
slat (usec): min=2, max=3537, avg= 3.09, stdev= 4.94
clat (nsec): min=450, max=3856.2k, avg=34526.94, stdev=32327.92
lat (usec): min=25, max=3862, avg=37.70, stdev=32.74
clat percentiles (usec):
| 1.00th=[ 26], 5.00th=[ 27], 10.00th=[ 27], 20.00th=[ 29],
| 30.00th=[ 29], 40.00th=[ 30], 50.00th=[ 31], 60.00th=[ 33],
| 70.00th=[ 35], 80.00th=[ 37], 90.00th=[ 39], 95.00th=[ 43],
| 99.00th=[ 118], 99.50th=[ 163], 99.90th=[ 243], 99.95th=[ 537],
| 99.99th=[ 1467]
bw ( KiB/s): min=43302, max=50888, per=100.00%, avg=47237.77, stdev=1569.59, samples=119
iops : min=10825, max=12722, avg=11809.37, stdev=392.40, samples=119
lat (nsec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=96.47%
lat (usec) : 100=2.19%, 250=1.20%, 500=0.04%, 750=0.02%, 1000=0.02%
lat (msec) : 2=0.02%, 4=0.01%
cpu : usr=1.39%, sys=13.28%, ctx=2104360, majf=0, minf=59077
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwt: total=0,708611,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=46.1MiB/s (48.4MB/s), 46.1MiB/s-46.1MiB/s (48.4MB/s-48.4MB/s), io=2768MiB (2902MB), run=60001-60001msec
Disk stats (read/write):
sdb: ios=15/1412648, merge=0/0, ticks=5/54262, in_queue=83332, util=99.71%
ВН
Дмитрий Дятлов
=== START OF INFORMATION SECTION ===
Device Model: Micron_5210_MTFDDAK7T6QDE
User Capacity: 7,681,501,126,656 bytes [7.68 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: Solid State Device
Form Factor: 2.5 inches
Device is: Not in smartctl database [for details use: -P showall]
ATA Version is: ACS-3 T13/2161-D revision 5
SATA Version is: SATA 3.2, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Jun 27 23:55:01 2020 MSK
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 256
On-line CPU(s) list: 0-255
Thread(s) per core: 2
Core(s) per socket: 64
Socket(s): 2
NUMA node(s): 2
Vendor ID: AuthenticAMD
CPU family: 23
Model: 49
Model name: AMD EPYC 7742 64-Core Processor
Stepping: 0
CPU MHz: 3248.343
CPU max MHz: 2250.0000
CPU min MHz: 1500.0000
BogoMIPS: 4500.43
Virtualization: AMD-V
L1d cache: 32K
L1i cache: 32K
L2 cache: 512K
L3 cache: 16384K
NUMA node0 CPU(s): 0-63,128-191
NUMA node1 CPU(s): 64-127,192-255
Device Model: Micron_5210_MTFDDAK7T6QDE
User Capacity: 7,681,501,126,656 bytes [7.68 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: Solid State Device
Form Factor: 2.5 inches
Device is: Not in smartctl database [for details use: -P showall]
ATA Version is: ACS-3 T13/2161-D revision 5
SATA Version is: SATA 3.2, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Jun 27 23:55:01 2020 MSK
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 256
On-line CPU(s) list: 0-255
Thread(s) per core: 2
Core(s) per socket: 64
Socket(s): 2
NUMA node(s): 2
Vendor ID: AuthenticAMD
CPU family: 23
Model: 49
Model name: AMD EPYC 7742 64-Core Processor
Stepping: 0
CPU MHz: 3248.343
CPU max MHz: 2250.0000
CPU min MHz: 1500.0000
BogoMIPS: 4500.43
Virtualization: AMD-V
L1d cache: 32K
L1i cache: 32K
L2 cache: 512K
L3 cache: 16384K
NUMA node0 CPU(s): 0-63,128-191
NUMA node1 CPU(s): 64-127,192-255
а, ну воткнут ещё куда. в материнку или в рейд контроллер какой то?
ВН
Дмитрий Дятлов
/dev/sdb:
setting drive write-caching to 1 (on)
write-caching = 1 (on)
[root@####### ~]# fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdb
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.1
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=47.3MiB/s][r=0,w=12.1k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=812725: Sun Jun 28 00:17:31 2020
write: IOPS=11.8k, BW=46.1MiB/s (48.4MB/s)(2768MiB/60001msec)
slat (usec): min=2, max=3537, avg= 3.09, stdev= 4.94
clat (nsec): min=450, max=3856.2k, avg=34526.94, stdev=32327.92
lat (usec): min=25, max=3862, avg=37.70, stdev=32.74
clat percentiles (usec):
| 1.00th=[ 26], 5.00th=[ 27], 10.00th=[ 27], 20.00th=[ 29],
| 30.00th=[ 29], 40.00th=[ 30], 50.00th=[ 31], 60.00th=[ 33],
| 70.00th=[ 35], 80.00th=[ 37], 90.00th=[ 39], 95.00th=[ 43],
| 99.00th=[ 118], 99.50th=[ 163], 99.90th=[ 243], 99.95th=[ 537],
| 99.99th=[ 1467]
bw ( KiB/s): min=43302, max=50888, per=100.00%, avg=47237.77, stdev=1569.59, samples=119
iops : min=10825, max=12722, avg=11809.37, stdev=392.40, samples=119
lat (nsec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=96.47%
lat (usec) : 100=2.19%, 250=1.20%, 500=0.04%, 750=0.02%, 1000=0.02%
lat (msec) : 2=0.02%, 4=0.01%
cpu : usr=1.39%, sys=13.28%, ctx=2104360, majf=0, minf=59077
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwt: total=0,708611,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=46.1MiB/s (48.4MB/s), 46.1MiB/s-46.1MiB/s (48.4MB/s-48.4MB/s), io=2768MiB (2902MB), run=60001-60001msec
Disk stats (read/write):
sdb: ios=15/1412648, merge=0/0, ticks=5/54262, in_queue=83332, util=99.71%
setting drive write-caching to 1 (on)
write-caching = 1 (on)
[root@####### ~]# fio -ioengine=libaio -fsync=1 -direct=1 -name=test -bs=4k -iodepth=1 -rw=randwrite -runtime=60 -filename=/dev/sdb
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.1
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=47.3MiB/s][r=0,w=12.1k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=812725: Sun Jun 28 00:17:31 2020
write: IOPS=11.8k, BW=46.1MiB/s (48.4MB/s)(2768MiB/60001msec)
slat (usec): min=2, max=3537, avg= 3.09, stdev= 4.94
clat (nsec): min=450, max=3856.2k, avg=34526.94, stdev=32327.92
lat (usec): min=25, max=3862, avg=37.70, stdev=32.74
clat percentiles (usec):
| 1.00th=[ 26], 5.00th=[ 27], 10.00th=[ 27], 20.00th=[ 29],
| 30.00th=[ 29], 40.00th=[ 30], 50.00th=[ 31], 60.00th=[ 33],
| 70.00th=[ 35], 80.00th=[ 37], 90.00th=[ 39], 95.00th=[ 43],
| 99.00th=[ 118], 99.50th=[ 163], 99.90th=[ 243], 99.95th=[ 537],
| 99.99th=[ 1467]
bw ( KiB/s): min=43302, max=50888, per=100.00%, avg=47237.77, stdev=1569.59, samples=119
iops : min=10825, max=12722, avg=11809.37, stdev=392.40, samples=119
lat (nsec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=96.47%
lat (usec) : 100=2.19%, 250=1.20%, 500=0.04%, 750=0.02%, 1000=0.02%
lat (msec) : 2=0.02%, 4=0.01%
cpu : usr=1.39%, sys=13.28%, ctx=2104360, majf=0, minf=59077
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwt: total=0,708611,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=46.1MiB/s (48.4MB/s), 46.1MiB/s-46.1MiB/s (48.4MB/s-48.4MB/s), io=2768MiB (2902MB), run=60001-60001msec
Disk stats (read/write):
sdb: ios=15/1412648, merge=0/0, ticks=5/54262, in_queue=83332, util=99.71%
ага. гуд
ДД
а, ну воткнут ещё куда. в материнку или в рейд контроллер какой то?
В материнку.
ДД
Ядро если что, 5.7.1-1.el7.elrepo
2020 June 29
DO
https://docs.ceph.com/docs/octopus/cephadm/install/
"Podman или Docker для запуска контейнеров" - зачем в рекомендациях это, если сам процесс инстала без докера
"Podman или Docker для запуска контейнеров" - зачем в рекомендациях это, если сам процесс инстала без докера
DO
DO
но если я не хочу использовать докер , а хочу что б оно все вертелось в systemd ... где блядь адекватный гайд)
DO
ебанутая документация
Am
Не используй цефадм
DO
Не используй цефадм
да , хорошая идея
DO
ceph-deploy ?
Am
ceph-deploy ?
Да
DO
ключи в каком порядке делаются ?