Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: a GPU driver issue is not the same as out of vram.

Browse Source
This commit is contained in:
Raimund58
2023-11-28 12:28:39 +00:00
committed by Sheepit Renderfarm
parent 1aa4ff3063
commit 199ee44671
2 changed files with 10 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/main/java/com/sheepit/client/Job.java
View File

@@ -843,106 +843,7 @@ import java.util.regex.Pattern;
// end of rendering
return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Launch exceeded timeout in")) {
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:08.44 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 2/24, Sample 10/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:07.08 | Mem:175.48M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 2/24, Sample 14/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:07.11 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 0/14
// CUDA error: Launch exceeded timeout in cuCtxSynchronize()
// Refer to the Cycles GPU rendering documentation for possible solutions:
// http://www.blender.org/manual/render/cycles/gpu_rendering.html
// CUDA error: Launch exceeded timeout in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size)
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:03:04.30 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 1/14
// CUDA error: Launch exceeded timeout in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size)
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:02:01.87 | Mem:175.48M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 14/14
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:02:01.87 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 4/24, Sample 0/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:27.05 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 4/24, Sample 14/14
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:00:00.75 | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 24/24, Sample 0/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 24/24, Sample 14/14
// Error: CUDA error: Launch exceeded timeout in cuCtxSynchronize()
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Cancel | CUDA error: Launch exceeded timeout in cuCtxSynchronize()
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 1-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 2-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 3-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 4-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 5-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 6-6
// Fra:420 Mem:109.00M (0.00M, Peak 215.18M) Sce: Scene Ve:0 Fa:0 La:0
// Saved: /tmp/xx/1234_0420.bmp Time: 00:18.29 (Saving: 00:00.06)
// Blender quit
// end of rendering
return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Invalid value in cuTexRefSetAddress(")) {
// Fra:83 Mem:1201.77M (0.00M, Peak 1480.94M) | Time:00:59.30 | Mem:894.21M, Peak:894.21M | color 3, RenderLayer | Updating Mesh | Copying Strands to device
// Fra:83 Mem:1316.76M (0.00M, Peak 1480.94M) | Time:01:02.84 | Mem:1010.16M, Peak:1010.16M | color 3, RenderLayer | Cancel | CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size)
// Error: CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size)
// Fra:83 Mem:136.82M (0.00M, Peak 1480.94M) | Time:01:03.40 | Sce: color 3 Ve:0 Fa:0 La:0
// Blender quit
// CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size)
// Refer to the Cycles GPU rendering documentation for possible solutions:
// http://www.blender.org/manual/render/cycles/gpu_rendering.html
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Launch failed in cuCtxSynchronize()")) {
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.95 | Remaining:00:03.88 | Mem:210.79M, Peak:210.79M | Scene, W Laser | Path Tracing Tile 16/18, Sample 36/36
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Remaining:00:00.82 | Mem:211.04M, Peak:211.04M | Scene, W Laser | Path Tracing Tile 17/18, Sample 36/36
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Mem:211.11M, Peak:211.11M | Scene, W Laser | Path Tracing Tile 18/18
// Error: CUDA error: Launch failed in cuCtxSynchronize(), line 1372
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Mem:211.11M, Peak:211.11M | Scene, W Laser | Cancel | CUDA error: Launch failed in cuCtxSynchronize(), line 1372
// Cycles shader graph connect: can only connect closure to closure (Invert.Color to Mix Shader.Closure1).
// Cycles shader graph connect: can only connect closure to closure (Mix Shader.Closure to Bump.Normal).
// CUDA error: Launch failed in cuCtxSynchronize(), line 1372
// Refer to the Cycles GPU rendering documentation for possible solutions:
// https://docs.blender.org/manual/en/dev/render/cycles/gpu_rendering.html
// CUDA error: Launch failed in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size), line 591
// CUDA error: Launch failed in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size), line 591
// CUDA error: Launch failed in cuMemFree(cuda_device_ptr(mem.device_pointer)), line 615
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) {
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.81 | Remaining:01:10:05.16 | Mem:175.14M, Peak:265.96M | Scene, RenderLayer | Path Tracing Tile 34/135, Sample 800/800, Denoised 17 tiles
// Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.81 | Remaining:01:10:04.95 | Mem:264.84M, Peak:266.90M | Scene, RenderLayer | Path Tracing Tile 34/135, Sample 800/800, Denoised 18 tiles
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:07:20.83 | Mem:266.90M, Peak:266.90M | Scene, RenderLayer | Path Tracing Tile 35/135, Sample 800/800, Denoised 18 tiles
// Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:07:20.63 | Mem:356.60M, Peak:358.67M | Scene, RenderLayer | Path Tracing Tile 35/135, Sample 800/800, Denoised 19 tiles
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:04:45.63 | Mem:358.67M, Peak:358.67M | Scene, RenderLayer | Path Tracing Tile 36/135, Sample 800/800, Denoised 19 tiles
// Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:04:45.45 | Mem:448.37M, Peak:450.43M | Scene, RenderLayer | Path Tracing Tile 36/135, Sample 800/800, Denoised 20 tiles
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.83 | Remaining:01:02:18.83 | Mem:450.43M, Peak:450.43M | Scene, RenderLayer | Path Tracing Tile 37/135, Sample 800/800, Denoised 20 tiles
// CUDA error: Illegal address in cuCtxSynchronize(), line 1372
// Refer to the Cycles GPU rendering documentation for possible solutions:
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("Error: Out of memory in CUDA")) {
else if (line.contains("Error: Out of memory in CUDA") || line.contains("Error: Out of memory in OPTIX")) {
// 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.27 | Mem:8411.20M, Peak:8411.20M | Scene, View Layer | Loading render kernels (may take a few minutes the first time)
// 15-03 21:13:45 (debug) Out of memory in CUDA queue enqueue (integrator_shade_surface)
// 15-03 21:13:45 (debug) Refer to the Cycles GPU rendering documentation for possible solutions:
@@ -954,7 +855,10 @@ import java.util.regex.Pattern;
// 15-03 21:13:46 (debug) Blender quit
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("Error: System is out of GPU memory") || line.contains("Error: Illegal address in CUDA") || line.contains("Error: Out of memory in cu") || line.contains("System is out of GPU memory")) {
else if (line.contains("CUDA error: Launch exceeded timeout in") || line.contains("CUDA error: Invalid value in cuTexRefSetAddress(") || line.contains("CUDA error: Launch failed in cuCtxSynchronize()") || line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) {
return Error.Type.GPU_OR_DRIVER_ERROR;
}
else if (line.contains("System is out of GPU memory")) {
// 16-03 18:37:37 (debug) Fra:192 Mem:15826.70M (Peak 15831.18M) | Time:02:21.86 | Mem:6622.87M, Peak:6625.35M | Scene, ViewLayer | Loading denoising kernels (may take a few minutes the first time)
// 16-03 18:37:37 (debug) System is out of GPU memory
// 16-03 18:37:37 (debug) Refer to the Cycles GPU rendering documentation for possible solutions:
@@ -966,7 +870,7 @@ import java.util.regex.Pattern;
// 16-03 18:37:40 (debug) end of rendering
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA device supported only with compute capability")) {
else if (line.contains("CUDA device supported only with compute capability") || line.contains("OPTIX device supported only with compute capability")) {
// found bundled python: /tmp/xx/2.73/python
// read blend: /tmp/xx/compute-method.blend
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun
@@ -1055,7 +959,7 @@ import java.util.regex.Pattern;
//09-01 09:27:33 (debug) Blender quit
return Error.Type.RENDERER_OUT_OF_MEMORY;
}
else if (line.contains("CUDA kernel compilation failed")) {
else if (line.contains("CUDA kernel compilation failed") || line.contains("OPTIX kernel compilation failed")) {
// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.001
// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.002
// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing