Troubleshoot slow/infinite `next build` behavior
Unanswered
sparr posted this in #help-forum
sparrOP
I am doing
The working environments are macos and linux, building in their native OS and arch (arm64 and amd64). The slow/stuck environment is docker+rosetta platform linux/amd64 hosted on arm64 macos.
next build --debug and getting stuck at info - Creating an optimized production build .. with the dots no longer animating. The process has been at 100% CPU for over an hour, for a build that takes <2 minutes in other environments. How can I get further insight into what it is doing right now?The working environments are macos and linux, building in their native OS and arch (arm64 and amd64). The slow/stuck environment is docker+rosetta platform linux/amd64 hosted on arm64 macos.
18 Replies
sparrOP
I just killed the Docker container after 24 hours of 100% cpu usage with no progress, and I still have no idea what it was doing.
My next idea is to run it under strace and see if any syscalls are happening during the churn
this is most likely a bug. any build runs taking more than 10 minutes in a small app (<2 mins normally) is abnormal and should be aborted, no need to wait a full day
please report this as a nextjs bug
and maybe use a different OS for development temporarily
(via github codespaces or similar services, for example)
sparrOP
strace -ff -o got me a TON of syscall tracing136 trace files, including the most active one growing at a rate of about 2GB per minute
the shell where I'm running yarn is nonresponsive once it enters the problematic state (identified by the
... animation stopping), including to ctrl+c and ctrl+z. it also doesn't respond to SIGTERM. I ended up using SIGKILL(I'll put this all in a bug report, just keeping track of things here in case someone can offer suggestions prior to the bug report)
sparrOP
here's where the big trace file goes off the rails:
syscall_0x8550320(0x3, 0x4cc10f0, 0x4ba0e20, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x4ba0de8(0x80, 0xca, 0xa636, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x80d44e0(0x3, 0x4cc10f0, 0x4ba0e20, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x6837728, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x6837580(0x4cc10f0, 0x3, 0x4ba0e20, 0, 0x4ba0dc0, 0x62) = 0
pread64(0, NULL, 0, 0) = 281473338258152
pread64(2560, NULL, 0, 0) = 281473338258152
syscall_0x5554ffff7530(0x5554ffff7330, 0x3c000, 0xffff97683000, 0, 0, 0xe2) = 0xffff9767b781
syscall_0x5554ffff7530(0x5554ffff7330, 0x3c000, 0xffff97503000, 0, 0, 0xe2) = 0xffff974e9020
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x82c2508, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x4ba0de8(0x80, 0xca, 0xa639, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x7f59148, 0, 0x4ba0dc0, 0x62) = 0
dup2(80482544, 133533688) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x80d8ba8, 0, 0x4ba0dc0, 0x62) = 0
pause() = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x6fe9988, 0, 0x4ba0dc0, 0x62) = 0
nanosleep({tv_sec=33642704, tv_nsec=140737484682256}, 0x6fe9838) = 0
syscall_0x4ba0de8(0x80, 0xca, 0xa63d, 0, 0x4ba0dc0, 0x62) = 0
alarm(80482544) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x75d8f28, 0, 0x4ba0dc0, 0x62) = 0
getitimer(0x4cc10f0 /* ITIMER_??? */, {it_interval={tv_sec=123571584, tv_usec=6}, it_value={tv_sec=5299989643273, tv_usec=2}}) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x4c95318, 0, 0x4ba0dc0, 0x62) = 0
setitimer(0x4cc10f0 /* ITIMER_??? */, {it_interval={tv_sec=80302448, tv_usec=6}, it_value={tv_sec=257, tv_usec=100434032}}, {it_interval=
{tv_sec=33750264, tv_usec=98381824}, it_value={tv_sec=80482544, tv_usec=0}}) = 0
syscall_0x4ba0de8(0x80, 0xca, 0xa640, 0, 0x4ba0dc0, 0x62) = 0
getpid() = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x809e7e8, 0, 0x4ba0dc0, 0x62) = 0sendfile(80482544, 134866584, [33750264], 0) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x7c8d1e8, 0, 0x4ba0dc0, 0x62) = 0
connect(80482544, {sa_family=0xd040 /* AF_??? */, sa_data="\310\7\0\0\0\0\6\0\0\0\0\0\0\0000\0\0\0\0\0\0\0\320\320\310\7\0\0\0\0\20\275,\
7\0\0\0\0\340\321\310\7\0\0\0\0`\322\310\7\0\0\0\0P\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0 \16\272\4\0\0\0\0\200\\\260\0\0\0\0\0\0\0\0\0\0\0\0\0\0
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0K\0\0\0\0\0\0\0\244\201\0\0\0\0\0\0"}, 130601024) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x8281e28, 0, 0x4ba0dc0, 0x62) = 0
accept(80482544, {sa_family=0x1c80 /* AF_??? */, sa_data="(\10\0\0\0\0\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0 \16\272\4\0\0\0\0\200\\\260\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0
\0\0\0\0\0K\0\0\0\0\0\0\0\244\201\0\0\0\0\0\0"}, [33750264]) = 0
syscall_0x4ba0de8(0x80, 0xca, 0xa644, 0, 0x4ba0dc0, 0x62) = 0
sendto(80482544, "\220u\364\7\0\0\0\0\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 133461392, 0, {sa_family=0x4c91 /* AF_??? */,
sa_data="\1\0\0\0\0\0\211L\1\0\0\0\0\0\6\0\0\0\0\0\0\0\3\0\0\0\0\0\0\0\20\0\0\0\32\0\0\0\0\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
0\0\0\0\0\0\0\0 \16\272\4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"}, 98) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x7f7c8c8, 0, 0x4ba0dc0, 0x62) = 0
recvfrom(80482544, "", 133678880, 0, 0x4ba0dc0, 0x62) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x747bcc8, 0, 0x4ba0dc0, 0x62) = 0
socket(0x4cc10f0 /* AF_??? */, SOCK_NONBLOCK|0x747b378, 122141472) = 0
syscall_0x4ba0de8(0x4cc10f0, 0x1, 0x69ebaf8, 0, 0x4ba0dc0, 0x62) = 0and then it starts producing gigabyte-long single lines that look like this:
sendmsg(80482544, {msg_name={sa_family=0xfcf8 /* AF_??? */, sa_data="\2\2\0\0"}, msg_namelen=6, msg_iov=[{iov_base=0x2000000070, iov_len=
177}, {iov_base="\0\0\0\0\0\0\0\0A\0\0\0\0\0\0\0P\335-\6\0\0\0\0G\0\0\0\0\0\0\0"..., iov_len=107487776}, {iov_base="&\0\0\0:\0\0\0h\0\0\0
o\0\0\0v\0\0\0e\0\0\0r\0\0\0,\0\0\0"..., iov_len=111065600}, {iov_base=0x30, iov_len=64}, {iov_base="", iov_len=0}, {iov_base=0x48, iov_l
en=72}, {iov_base=NULL, iov_len=284}, {iov_base=0x10, iov_len=65}, {iov_base="0\0\0\0\0\0\0\0A\0\0\0\0\0\0\0\20P\314\7\0\0\0\0\266\6\0\0\
0\0\0\0"..., iov_len=110753520}, {iov_base=0x4c, iov_len=76}, {iov_base=NULL, iov_len=80397408}, {iov_base=0x100, iov_len=96}, {iov_base=
"\0\0\0\0\0\0\0\0!\2\0\0\0\0\0\0\220\362\21\10\0\0\0\0\20\200\273\4\0\0\0\0"..., iov_len=498216206446}, {iov_base=0x700000002d, iov_len=4sparrOP
The problem only occurs specifically in a linux/amd64 docker container on a macos arm64 docker host (which is definitely using rosetta and may be doing other emulation). building works fine on native linux/amd64 and macos/arm64, and on linux/amd64 container on linux/amd64, and on linux/arm64 container on macos arm64 host
sparrOP
I reported this as a bug in Docker Desktop for Mac https://github.com/docker/for-mac/issues/7124
sparrOP
wow, bot closed my issue, so now I'm tracking down the source of the bot to find out why...