inode 没满,但是会随机报磁盘空间不足是啥问题?

V2EX = way to explore

V2EX 是一个关于分享和探索的地方

现在注册

已注册用户请登录

Distributions

› Ubuntu

› Fedora

› CentOS

中文资源站

› 网易开源镜像站

这是一个创建于 804 天前的主题，其中的信息可能已经有所发展或是发生改变。

创建一个文件 123456789,可能会报磁盘空间不足这时我把名字改成 123456789a 可能就会创建成功但是也有可能还是失败但是只要我继续改多试几次就一定会成功

报磁盘

创建

inode

不足

18 条回复 • 2022-02-12 10:36:39 +08:00

huangmingyou

2022-02-11 14:33:40 +08:00

strace mkdir 123456789 看看

duke807

2022-02-11 14:41:31 +08:00 via Android

inode 使用的數量和硬盤使用的空間沒有絕對的關係
建立文件系統的時候，使用默認推薦的 inode 數量的話，如果存放超多小文件，最後會導致空間還剩下很多，但是由於沒有多餘 inode 導致無法建立新文件

你说的隨機估計是有其他程序在讀寫同一個文件系統吧

FenixVu

2022-02-11 17:06:31 +08:00

@duke807 我这边确实都是小文件差不多 1000w,问题是我现在磁盘空间也有 inode 也有,我这边是一个 minio 的存储应该只有他在写文件

FenixVu

2022-02-11 17:07:07 +08:00

@huangmingyou 看不太懂

```
execve("/bin/mkdir", ["mkdir", ".minio.sys/buckets/nethospitalse"...], [/* 20 vars */]) = 0
brk(NULL) = 0x12bf000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=35801, ...}) = 0
mmap(NULL, 35801, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbe711e3000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260Z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=130224, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711e2000
mmap(NULL, 2234080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe70da5000
mprotect(0x7fbe70dc4000, 2093056, PROT_NONE) = 0
mmap(0x7fbe70fc3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7fbe70fc3000
mmap(0x7fbe70fc5000, 5856, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbe70fc5000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe709db000
mprotect(0x7fbe70b9b000, 2097152, PROT_NONE) = 0
mmap(0x7fbe70d9b000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7fbe70d9b000
mmap(0x7fbe70da1000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbe70da1000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpcre.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=456632, ...}) = 0
mmap(NULL, 2552072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe7076b000
mprotect(0x7fbe707d9000, 2097152, PROT_NONE) = 0
mmap(0x7fbe709d9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6e000) = 0x7fbe709d9000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14608, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711e1000
mmap(NULL, 2109680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe70567000
mprotect(0x7fbe7056a000, 2093056, PROT_NONE) = 0
mmap(0x7fbe70769000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fbe70769000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260`\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=138696, ...}) = 0
mmap(NULL, 2212904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe7034a000
mprotect(0x7fbe70362000, 2093056, PROT_NONE) = 0
mmap(0x7fbe70561000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fbe70561000
mmap(0x7fbe70563000, 13352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbe70563000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711e0000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711de000
arch_prctl(ARCH_SET_FS, 0x7fbe711de800) = 0
mprotect(0x7fbe70d9b000, 16384, PROT_READ) = 0
mprotect(0x7fbe70561000, 4096, PROT_READ) = 0
mprotect(0x7fbe70769000, 4096, PROT_READ) = 0
mprotect(0x7fbe709d9000, 4096, PROT_READ) = 0
mprotect(0x7fbe70fc3000, 4096, PROT_READ) = 0
mprotect(0x611000, 4096, PROT_READ) = 0
mprotect(0x7fbe711ec000, 4096, PROT_READ) = 0
munmap(0x7fbe711e3000, 35801) = 0
set_tid_address(0x7fbe711dead0) = 62296
set_robust_list(0x7fbe711deae0, 24) = 0
rt_sigaction(SIGRTMIN, {0x7fbe7034fb50, [], SA_RESTORER|SA_SIGINFO, 0x7fbe7035b390}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7fbe7034fbe0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7fbe7035b390}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffe96852140) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffe96852140) = -1 ENOENT (No such file or directory)
brk(NULL) = 0x12bf000
brk(0x12e0000) = 0x12e0000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 392
read(3, "", 1024) = 0
close(3) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1668976, ...}) = 0
mmap(NULL, 1668976, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbe71046000
close(3) = 0
mkdir(".minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml", 0777) = -1 ENOSPC (No space left on device)
open("/usr/lib/x86_64-linux-gnu/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mkdir: ", 7mkdir: ) = 7
write(2, "cannot create directory \342\200\230.mini"..., 129cannot create directory ‘.minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml’) = 129
write(2, ": No space left on device", 25: No space left on device) = 25
write(2, "\n", 1
) = 1
close(1) = 0
close(2) = 0
exit_group(1) = ?
+++ exited with 1 +++
```

duke807

2022-02-11 17:31:50 +08:00 via Android

@FenixVu 空間和 inode 的使用情況，你分別貼出來讓大家看一下，譬如總共多少，已使用多少

查看所用分區空間使用情況的命令是 df
查看 inode 不常用記不住，上網查一下是什麼命令

要進一步排除是否有第三方軟件讀寫的話，linux 也有監控指定路徑讀寫情況的機制

defunct9

2022-02-11 17:42:09 +08:00

df -i

FenixVu

2022-02-11 17:56:55 +08:00

@duke807
Filesystem Type Size Used Avail Use% Mounted on
udev devtmpfs 16G 0 16G 0% /dev
tmpfs tmpfs 3.2G 338M 2.9G 11% /run
/dev/mapper/ubuntu--vg-root ext4 1.5T 700G 745G 49% /
tmpfs tmpfs 16G 4.0K 16G 1% /dev/shm
tmpfs tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs tmpfs 16G 0 16G 0% /sys/fs/cgroup
/dev/sda1 ext2 720M 58M 626M 9% /boot
/home/hisee/.Private ecryptfs 1.5T 700G 745G 49% /home/hisee
overlay overlay 1.5T 700G 745G 49%

Filesystem Inodes IUsed IFree IUse% Mounted on
udev 4110911 430 4110481 1% /dev
tmpfs 4116178 865 4115313 1% /run
/dev/mapper/ubuntu--vg-root 100425728 35748318 64677410 36% /
tmpfs 4116178 2 4116176 1% /dev/shm
tmpfs 4116178 3 4116175 1% /run/lock
tmpfs 4116178 16 4116162 1% /sys/fs/cgroup
/dev/sda1 46848 299 46549 1% /boot
/home/hisee/.Private 100425728 35748318 64677410 36% /home/hisee
overlay 100425728 35748318 64677410 36% /var/lib/docker/overlay2/8bdfd3b8eb24e7422778b6e6ecc13a900bfa12bbad903fc283694d0148b511ca/merged

feedcode

2022-02-11 19:35:59 +08:00

遇到了哈希冲突，
ext4 uses half_md4 as a default hashing-mechanism. If I interpret my google-results correctly, this uses the md4-hash algorithm, but strips it to 32 bits.

https://blog.merovius.de/2013/10/20/ext4-mysterious-no-space-left-on.html

zhoudaiyu

2022-02-11 19:56:52 +08:00

@feedcode #8
@FenixVu #3 lz 贴个"lsblk -f"的结果呗

zhoudaiyu

2022-02-11 19:58:18 +08:00

@zhoudaiyu #9 看到了 lz 贴的 df 的结果了，抱歉

zhoudaiyu

2022-02-11 19:58:56 +08:00

@feedcode #8 这都行，学到了！

liuxu

2022-02-11 20:20:02 +08:00

@FenixVu #4

结合楼主之前的帖子和 strace ，是因为你测试的目录是 minio 所管理的空间，然后 minio 服务器使用的.minio.sys
/目录磁盘爆了

mkdir(".minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml", 0777) = -1 ENOSPC (No space left on device)
open("/usr/lib/x86_64-linux-gnu/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mkdir: ", 7mkdir: ) = 7
write(2, "cannot create directory \342\200\230.mini"..., 129cannot create directory ‘.minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml’) = 129
write(2, ": No space left on device", 25: No space left on device) = 25
write(2, "\n", 1
) = 1

创建.minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml 没空间了

给你看看我的 strace

liuxu@liuxu-Inspiron-7559:/tmp$ strace mkdir 123456789
execve("/usr/bin/mkdir", ["mkdir", "123456789"], 0x7ffd98b5c8c8 /* 60 vars */) = 0
brk(NULL) = 0x5568f5d55000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffc7dca4820) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
...
...
access("/etc/selinux/config", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=14537584, ...}) = 0
mmap(NULL, 14537584, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0d0e1e000
close(3) = 0
mkdir("123456789", 0777) = 0
close(1) = 0
close(2) = 0
exit_group(0) = ?
+++ exited with 0 +++

最下面 access("/etc/selinux/config", F_OK) 后直接 mkdir("123456789", 0777) ，楼主的连 mkdir("123456789", 0777)都没执行

楼上的大哥们虽然都是好心，但是别乱指路啊

duke807

2022-02-11 20:46:42 +08:00 via Android

沒看出來樓主 .minio.sys 目錄在哪個路徑，應該是在 overlay 管轄的 home 分區

不知道 #8 提到的問題是否會影響 ext4 之上的 overlay ，我覺得應該會影響

反正 #8 提供的信息對我有幫助，我就給讚了
反而 #12 的觀點我不認同

liuxu

2022-02-11 21:54:51 +08:00

@duke807 #13
8 楼的也有可能 ext4 问题，创建文件夹对于 minio 对象系统来说可能就是只创建一个 xml 文件记录一下，而不会调用 mkdir 123456789

但如果是 hash 导致的问题，很难解释通创建 123456789 失败，而创建 123456789a 却可以间歇成功
应该是不管是 123456789 还是 123456789a ，都一直失败或都间歇失败

liuxu

2022-02-11 22:00:29 +08:00

要是 8 楼的还不行，看看 inode 是不是没用完

$ df . | sed -n '$s/ .*//p' | xargs sudo dumpe2fs -h

documentzhangx66

2022-02-12 02:59:31 +08:00

这个问题其实很简单。

你这盘又没多大，目前不确定到底是 ext4 还是 minio 的问题，那就排除法嘛。

先做一个同样大小、同样 ext4 参数的盘，把 ext4 dd 过去，用同样的系统挂载，再创建文件，看看会不会失败。

如果失败，说明是 ext4 的问题，如果成功，那就说明是 minio 的问题。

接下来就很简单了。

如果是 ext4 的问题，调试 ext4 ；如果是 minio 的问题，调试 minio 。

levinit

2022-02-12 08:54:54 +08:00 via iPhone

@feedcode 学习了留个心眼

FenixVu

2022-02-12 10:36:39 +08:00

@feedcode 我开始也考虑过是哈希问题,但是我想的方向是底层硬盘某一块满了,去查看后发现并没有,没往文件系统这块想,感谢你分享的文章看完后发现只能更换文件系统类型了