Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'cleanup/vram-errors' into 'master'

Browse Source 
Remove VRAM errors/Add Optix errors

See merge request sheepitrenderfarm/client!264
This commit is contained in:
Sheepit Renderfarm
2023-11-28 12:28:41 +00:00
parent 1aa4ff3063 199ee44671
commit 49e4baaf93
2 changed files with 10 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/sheepit/client/Error.java
View File

@@ -49,6 +49,7 @@ public class Error {
OS_NOT_SUPPORTED(17), OS_NOT_SUPPORTED(17),
CPU_NOT_SUPPORTED(18), CPU_NOT_SUPPORTED(18),
GPU_NOT_SUPPORTED(19), GPU_NOT_SUPPORTED(19),
GPU_OR_DRIVER_ERROR(32),
ENGINE_NOT_AVAILABLE(27), ENGINE_NOT_AVAILABLE(27),
VALIDATION_FAILED(25), VALIDATION_FAILED(25),
IMAGE_WRONG_DIMENSION(26), IMAGE_WRONG_DIMENSION(26),
@@ -228,6 +229,8 @@ public class Error {
return "Blender failed to read the project files. Will try another project in a few minutes."; return "Blender failed to read the project files. Will try another project in a few minutes.";
case DETECT_DEVICE_ERROR: case DETECT_DEVICE_ERROR:
return "Blender failed to detect the render device. Please check if you have all the necessary libraries installed and if your drivers are up to date."; return "Blender failed to detect the render device. Please check if you have all the necessary libraries installed and if your drivers are up to date.";
case GPU_OR_DRIVER_ERROR:
return "There was an error with your GPU or driver. Please ensure you have the latest driver and don't run extreme over/underclockings.";
default: default:
return in.toString(); return in.toString();
} }

110
src/main/java/com/sheepit/client/Job.java
View File

@@ -843,106 +843,7 @@ import java.util.regex.Pattern;
// end of rendering // end of rendering
return Type.RENDERER_OUT_OF_VIDEO_MEMORY; return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
} }
else if (line.contains("CUDA error: Launch exceeded timeout in")) { else if (line.contains("Error: Out of memory in CUDA") || line.contains("Error: Out of memory in OPTIX")) {
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:08.44 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 2/24, Sample 10/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:07.08 | Mem:175.48M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 2/24, Sample 14/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:07.11 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 0/14
// CUDA error: Launch exceeded timeout in cuCtxSynchronize()
// Refer to the Cycles GPU rendering documentation for possible solutions:
// http://www.blender.org/manual/render/cycles/gpu_rendering.html
// CUDA error: Launch exceeded timeout in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size)
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:03:04.30 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 1/14
// CUDA error: Launch exceeded timeout in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size)
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:02:01.87 | Mem:175.48M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 14/14
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:02:01.87 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 4/24, Sample 0/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:27.05 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 4/24, Sample 14/14
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size)
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:00:00.75 | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 24/24, Sample 0/14
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 24/24, Sample 14/14
// Error: CUDA error: Launch exceeded timeout in cuCtxSynchronize()
// Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Cancel | CUDA error: Launch exceeded timeout in cuCtxSynchronize()
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer))
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 1-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 2-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 3-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 4-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 5-6
// Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 6-6
// Fra:420 Mem:109.00M (0.00M, Peak 215.18M) Sce: Scene Ve:0 Fa:0 La:0
// Saved: /tmp/xx/1234_0420.bmp Time: 00:18.29 (Saving: 00:00.06)
// Blender quit
// end of rendering
return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Invalid value in cuTexRefSetAddress(")) {
// Fra:83 Mem:1201.77M (0.00M, Peak 1480.94M) | Time:00:59.30 | Mem:894.21M, Peak:894.21M | color 3, RenderLayer | Updating Mesh | Copying Strands to device
// Fra:83 Mem:1316.76M (0.00M, Peak 1480.94M) | Time:01:02.84 | Mem:1010.16M, Peak:1010.16M | color 3, RenderLayer | Cancel | CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size)
// Error: CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size)
// Fra:83 Mem:136.82M (0.00M, Peak 1480.94M) | Time:01:03.40 | Sce: color 3 Ve:0 Fa:0 La:0
// Blender quit
// CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size)
// Refer to the Cycles GPU rendering documentation for possible solutions:
// http://www.blender.org/manual/render/cycles/gpu_rendering.html
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Launch failed in cuCtxSynchronize()")) {
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.95 | Remaining:00:03.88 | Mem:210.79M, Peak:210.79M | Scene, W Laser | Path Tracing Tile 16/18, Sample 36/36
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Remaining:00:00.82 | Mem:211.04M, Peak:211.04M | Scene, W Laser | Path Tracing Tile 17/18, Sample 36/36
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Mem:211.11M, Peak:211.11M | Scene, W Laser | Path Tracing Tile 18/18
// Error: CUDA error: Launch failed in cuCtxSynchronize(), line 1372
// Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Mem:211.11M, Peak:211.11M | Scene, W Laser | Cancel | CUDA error: Launch failed in cuCtxSynchronize(), line 1372
// Cycles shader graph connect: can only connect closure to closure (Invert.Color to Mix Shader.Closure1).
// Cycles shader graph connect: can only connect closure to closure (Mix Shader.Closure to Bump.Normal).
// CUDA error: Launch failed in cuCtxSynchronize(), line 1372
// Refer to the Cycles GPU rendering documentation for possible solutions:
// https://docs.blender.org/manual/en/dev/render/cycles/gpu_rendering.html
// CUDA error: Launch failed in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size), line 591
// CUDA error: Launch failed in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size), line 591
// CUDA error: Launch failed in cuMemFree(cuda_device_ptr(mem.device_pointer)), line 615
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) {
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.81 | Remaining:01:10:05.16 | Mem:175.14M, Peak:265.96M | Scene, RenderLayer | Path Tracing Tile 34/135, Sample 800/800, Denoised 17 tiles
// Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.81 | Remaining:01:10:04.95 | Mem:264.84M, Peak:266.90M | Scene, RenderLayer | Path Tracing Tile 34/135, Sample 800/800, Denoised 18 tiles
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:07:20.83 | Mem:266.90M, Peak:266.90M | Scene, RenderLayer | Path Tracing Tile 35/135, Sample 800/800, Denoised 18 tiles
// Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:07:20.63 | Mem:356.60M, Peak:358.67M | Scene, RenderLayer | Path Tracing Tile 35/135, Sample 800/800, Denoised 19 tiles
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:04:45.63 | Mem:358.67M, Peak:358.67M | Scene, RenderLayer | Path Tracing Tile 36/135, Sample 800/800, Denoised 19 tiles
// Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:04:45.45 | Mem:448.37M, Peak:450.43M | Scene, RenderLayer | Path Tracing Tile 36/135, Sample 800/800, Denoised 20 tiles
// Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.83 | Remaining:01:02:18.83 | Mem:450.43M, Peak:450.43M | Scene, RenderLayer | Path Tracing Tile 37/135, Sample 800/800, Denoised 20 tiles
// CUDA error: Illegal address in cuCtxSynchronize(), line 1372
// Refer to the Cycles GPU rendering documentation for possible solutions:
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
}
else if (line.contains("Error: Out of memory in CUDA")) {
// 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.27 | Mem:8411.20M, Peak:8411.20M | Scene, View Layer | Loading render kernels (may take a few minutes the first time) // 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.27 | Mem:8411.20M, Peak:8411.20M | Scene, View Layer | Loading render kernels (may take a few minutes the first time)
// 15-03 21:13:45 (debug) Out of memory in CUDA queue enqueue (integrator_shade_surface) // 15-03 21:13:45 (debug) Out of memory in CUDA queue enqueue (integrator_shade_surface)
// 15-03 21:13:45 (debug) Refer to the Cycles GPU rendering documentation for possible solutions: // 15-03 21:13:45 (debug) Refer to the Cycles GPU rendering documentation for possible solutions:
@@ -954,7 +855,10 @@ import java.util.regex.Pattern;
// 15-03 21:13:46 (debug) Blender quit // 15-03 21:13:46 (debug) Blender quit
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
} }
else if (line.contains("Error: System is out of GPU memory") || line.contains("Error: Illegal address in CUDA") || line.contains("Error: Out of memory in cu") || line.contains("System is out of GPU memory")) { else if (line.contains("CUDA error: Launch exceeded timeout in") || line.contains("CUDA error: Invalid value in cuTexRefSetAddress(") || line.contains("CUDA error: Launch failed in cuCtxSynchronize()") || line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) {
return Error.Type.GPU_OR_DRIVER_ERROR;
}
else if (line.contains("System is out of GPU memory")) {
// 16-03 18:37:37 (debug) Fra:192 Mem:15826.70M (Peak 15831.18M) | Time:02:21.86 | Mem:6622.87M, Peak:6625.35M | Scene, ViewLayer | Loading denoising kernels (may take a few minutes the first time) // 16-03 18:37:37 (debug) Fra:192 Mem:15826.70M (Peak 15831.18M) | Time:02:21.86 | Mem:6622.87M, Peak:6625.35M | Scene, ViewLayer | Loading denoising kernels (may take a few minutes the first time)
// 16-03 18:37:37 (debug) System is out of GPU memory // 16-03 18:37:37 (debug) System is out of GPU memory
// 16-03 18:37:37 (debug) Refer to the Cycles GPU rendering documentation for possible solutions: // 16-03 18:37:37 (debug) Refer to the Cycles GPU rendering documentation for possible solutions:
@@ -966,7 +870,7 @@ import java.util.regex.Pattern;
// 16-03 18:37:40 (debug) end of rendering // 16-03 18:37:40 (debug) end of rendering
return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
} }
else if (line.contains("CUDA device supported only with compute capability")) { else if (line.contains("CUDA device supported only with compute capability") || line.contains("OPTIX device supported only with compute capability")) {
// found bundled python: /tmp/xx/2.73/python // found bundled python: /tmp/xx/2.73/python
// read blend: /tmp/xx/compute-method.blend // read blend: /tmp/xx/compute-method.blend
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun // Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun
@@ -1055,7 +959,7 @@ import java.util.regex.Pattern;
//09-01 09:27:33 (debug) Blender quit //09-01 09:27:33 (debug) Blender quit
return Error.Type.RENDERER_OUT_OF_MEMORY; return Error.Type.RENDERER_OUT_OF_MEMORY;
} }
else if (line.contains("CUDA kernel compilation failed")) { else if (line.contains("CUDA kernel compilation failed") || line.contains("OPTIX kernel compilation failed")) {
// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.001 // Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.001
// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.002 // Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.002
// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing // Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing