diff --git a/src/main/java/com/sheepit/client/Error.java b/src/main/java/com/sheepit/client/Error.java index 9cf75d2..103e784 100644 --- a/src/main/java/com/sheepit/client/Error.java +++ b/src/main/java/com/sheepit/client/Error.java @@ -49,6 +49,7 @@ public class Error { OS_NOT_SUPPORTED(17), CPU_NOT_SUPPORTED(18), GPU_NOT_SUPPORTED(19), + GPU_OR_DRIVER_ERROR(32), ENGINE_NOT_AVAILABLE(27), VALIDATION_FAILED(25), IMAGE_WRONG_DIMENSION(26), @@ -228,6 +229,8 @@ public class Error { return "Blender failed to read the project files. Will try another project in a few minutes."; case DETECT_DEVICE_ERROR: return "Blender failed to detect the render device. Please check if you have all the necessary libraries installed and if your drivers are up to date."; + case GPU_OR_DRIVER_ERROR: + return "There was an error with your GPU or driver. Please ensure you have the latest driver and don't run extreme over/underclockings."; default: return in.toString(); } diff --git a/src/main/java/com/sheepit/client/Job.java b/src/main/java/com/sheepit/client/Job.java index 079f058..e7829e1 100644 --- a/src/main/java/com/sheepit/client/Job.java +++ b/src/main/java/com/sheepit/client/Job.java @@ -843,106 +843,7 @@ import java.util.regex.Pattern; // end of rendering return Type.RENDERER_OUT_OF_VIDEO_MEMORY; } - else if (line.contains("CUDA error: Launch exceeded timeout in")) { - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:08.44 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 2/24, Sample 10/14 - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:07.08 | Mem:175.48M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 2/24, Sample 14/14 - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:07.11 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 0/14 - // CUDA error: Launch exceeded timeout in cuCtxSynchronize() - // Refer to the Cycles GPU rendering documentation for possible solutions: - // http://www.blender.org/manual/render/cycles/gpu_rendering.html - // CUDA error: Launch exceeded timeout in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size) - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:03:04.30 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 1/14 - // CUDA error: Launch exceeded timeout in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:02:01.87 | Mem:175.48M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 3/24, Sample 14/14 - // CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size) - // CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size) - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:02:01.87 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 4/24, Sample 0/14 - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:01:27.05 | Mem:176.04M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 4/24, Sample 14/14 - // CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size) - // CUDA error: Launch exceeded timeout in cuMemAlloc(&device_pointer, size) - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Remaining:00:00.75 | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 24/24, Sample 0/14 - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Path Tracing Tile 24/24, Sample 14/14 - // Error: CUDA error: Launch exceeded timeout in cuCtxSynchronize() - // Fra:420 Mem:102.41M (0.00M, Peak 215.18M) | Mem:185.66M, Peak:199.23M | Scene, RenderLayer | Cancel | CUDA error: Launch exceeded timeout in cuCtxSynchronize() - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // CUDA error: Launch exceeded timeout in cuMemFree(cuda_device_ptr(mem.device_pointer)) - // Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 1-6 - // Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 2-6 - // Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 3-6 - // Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 4-6 - // Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 5-6 - // Mem:109.00M (0.00M, Peak 215.18M) | Elapsed 00:00.00 | Tree Compositing Nodetree, Tile 6-6 - // Fra:420 Mem:109.00M (0.00M, Peak 215.18M) Sce: Scene Ve:0 Fa:0 La:0 - // Saved: /tmp/xx/1234_0420.bmp Time: 00:18.29 (Saving: 00:00.06) - // Blender quit - // end of rendering - return Type.RENDERER_OUT_OF_VIDEO_MEMORY; - } - else if (line.contains("CUDA error: Invalid value in cuTexRefSetAddress(")) { - // Fra:83 Mem:1201.77M (0.00M, Peak 1480.94M) | Time:00:59.30 | Mem:894.21M, Peak:894.21M | color 3, RenderLayer | Updating Mesh | Copying Strands to device - // Fra:83 Mem:1316.76M (0.00M, Peak 1480.94M) | Time:01:02.84 | Mem:1010.16M, Peak:1010.16M | color 3, RenderLayer | Cancel | CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size) - // Error: CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size) - // Fra:83 Mem:136.82M (0.00M, Peak 1480.94M) | Time:01:03.40 | Sce: color 3 Ve:0 Fa:0 La:0 - // Blender quit - // CUDA error: Invalid value in cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size) - // Refer to the Cycles GPU rendering documentation for possible solutions: - // http://www.blender.org/manual/render/cycles/gpu_rendering.html - return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; - } - else if (line.contains("CUDA error: Launch failed in cuCtxSynchronize()")) { - // Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.95 | Remaining:00:03.88 | Mem:210.79M, Peak:210.79M | Scene, W Laser | Path Tracing Tile 16/18, Sample 36/36 - // Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Remaining:00:00.82 | Mem:211.04M, Peak:211.04M | Scene, W Laser | Path Tracing Tile 17/18, Sample 36/36 - // Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Mem:211.11M, Peak:211.11M | Scene, W Laser | Path Tracing Tile 18/18 - // Error: CUDA error: Launch failed in cuCtxSynchronize(), line 1372 - // Fra:60 Mem:278.24M (0.00M, Peak 644.01M) | Time:05:08.96 | Mem:211.11M, Peak:211.11M | Scene, W Laser | Cancel | CUDA error: Launch failed in cuCtxSynchronize(), line 1372 - // Cycles shader graph connect: can only connect closure to closure (Invert.Color to Mix Shader.Closure1). - // Cycles shader graph connect: can only connect closure to closure (Mix Shader.Closure to Bump.Normal). - // CUDA error: Launch failed in cuCtxSynchronize(), line 1372 - // Refer to the Cycles GPU rendering documentation for possible solutions: - // https://docs.blender.org/manual/en/dev/render/cycles/gpu_rendering.html - // CUDA error: Launch failed in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size), line 591 - // CUDA error: Launch failed in cuMemcpyDtoH((uchar*)mem.data_pointer + offset, (CUdeviceptr)(mem.device_pointer + offset), size), line 591 - // CUDA error: Launch failed in cuMemFree(cuda_device_ptr(mem.device_pointer)), line 615 - return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; - } - else if (line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) { - // Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.81 | Remaining:01:10:05.16 | Mem:175.14M, Peak:265.96M | Scene, RenderLayer | Path Tracing Tile 34/135, Sample 800/800, Denoised 17 tiles - // Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.81 | Remaining:01:10:04.95 | Mem:264.84M, Peak:266.90M | Scene, RenderLayer | Path Tracing Tile 34/135, Sample 800/800, Denoised 18 tiles - // Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:07:20.83 | Mem:266.90M, Peak:266.90M | Scene, RenderLayer | Path Tracing Tile 35/135, Sample 800/800, Denoised 18 tiles - // Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:07:20.63 | Mem:356.60M, Peak:358.67M | Scene, RenderLayer | Path Tracing Tile 35/135, Sample 800/800, Denoised 19 tiles - // Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:04:45.63 | Mem:358.67M, Peak:358.67M | Scene, RenderLayer | Path Tracing Tile 36/135, Sample 800/800, Denoised 19 tiles - // Fra:124 Mem:432.71M (0.00M, Peak 435.34M) | Time:25:50.82 | Remaining:01:04:45.45 | Mem:448.37M, Peak:450.43M | Scene, RenderLayer | Path Tracing Tile 36/135, Sample 800/800, Denoised 20 tiles - // Fra:124 Mem:434.77M (0.00M, Peak 435.34M) | Time:25:50.83 | Remaining:01:02:18.83 | Mem:450.43M, Peak:450.43M | Scene, RenderLayer | Path Tracing Tile 37/135, Sample 800/800, Denoised 20 tiles - // CUDA error: Illegal address in cuCtxSynchronize(), line 1372 - // Refer to the Cycles GPU rendering documentation for possible solutions: - return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; - } - else if (line.contains("Error: Out of memory in CUDA")) { + else if (line.contains("Error: Out of memory in CUDA") || line.contains("Error: Out of memory in OPTIX")) { // 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.27 | Mem:8411.20M, Peak:8411.20M | Scene, View Layer | Loading render kernels (may take a few minutes the first time) // 15-03 21:13:45 (debug) Out of memory in CUDA queue enqueue (integrator_shade_surface) // 15-03 21:13:45 (debug) Refer to the Cycles GPU rendering documentation for possible solutions: @@ -954,7 +855,10 @@ import java.util.regex.Pattern; // 15-03 21:13:46 (debug) Blender quit return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; } - else if (line.contains("Error: System is out of GPU memory") || line.contains("Error: Illegal address in CUDA") || line.contains("Error: Out of memory in cu") || line.contains("System is out of GPU memory")) { + else if (line.contains("CUDA error: Launch exceeded timeout in") || line.contains("CUDA error: Invalid value in cuTexRefSetAddress(") || line.contains("CUDA error: Launch failed in cuCtxSynchronize()") || line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) { + return Error.Type.GPU_OR_DRIVER_ERROR; + } + else if (line.contains("System is out of GPU memory")) { // 16-03 18:37:37 (debug) Fra:192 Mem:15826.70M (Peak 15831.18M) | Time:02:21.86 | Mem:6622.87M, Peak:6625.35M | Scene, ViewLayer | Loading denoising kernels (may take a few minutes the first time) // 16-03 18:37:37 (debug) System is out of GPU memory // 16-03 18:37:37 (debug) Refer to the Cycles GPU rendering documentation for possible solutions: @@ -966,7 +870,7 @@ import java.util.regex.Pattern; // 16-03 18:37:40 (debug) end of rendering return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY; } - else if (line.contains("CUDA device supported only with compute capability")) { + else if (line.contains("CUDA device supported only with compute capability") || line.contains("OPTIX device supported only with compute capability")) { // found bundled python: /tmp/xx/2.73/python // read blend: /tmp/xx/compute-method.blend // Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun @@ -1055,7 +959,7 @@ import java.util.regex.Pattern; //09-01 09:27:33 (debug) Blender quit return Error.Type.RENDERER_OUT_OF_MEMORY; } - else if (line.contains("CUDA kernel compilation failed")) { + else if (line.contains("CUDA kernel compilation failed") || line.contains("OPTIX kernel compilation failed")) { // Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.001 // Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.002 // Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing