{"caller":"compact.go:1291","component":"compactor","level":"info","msg":"start sync of metas","org_id":"GC","ts":"2023-10-17T08:04:48.841909267Z"}{"caller":"fetcher.go:327","component":"block.BaseFetcher","concurrency":20,"level":"debug","msg":"fetching meta data","org_id":"GC","ts":"2023-10-17T08:04:48.842241761Z"}{"cached":1124,"caller":"fetcher.go:478","component":"block.BaseFetcher","duration":"1.404187325s","duration_ms":1404,"level":"info","msg":"successfully synchronized block metadata","org_id":"GC","partial":0,"returned":1123,"ts":"2023-10-17T08:04:50.246310867Z"}{"caller":"compact.go:1296","component":"compactor","level":"info","msg":"start of GC","org_id":"GC","ts":"2023-10-17T08:04:50.246405036Z"}{"caller":"compact.go:1319","component":"compactor","level":"info","msg":"start of compactions","org_id":"GC","ts":"2023-10-17T08:04:50.344171068Z"}{"caller":"compact.go:1005","component":"compactor","group":"0@{__org_id__=\"GC\"}","groupKey":"0@7253914978157373696","level":"info","msg":"compaction available and planned; downloading blocks","org_id":"GC","plan":"[01HCW6SYRH47K26KZS2H3K3H18 (min time: 1697414400000, max time: 1697457600000) 01HCW98DQ8Z3A1VF9ZF71SYJZX (min time: 1697450400000, max time: 1697457600000) 01HCW9AQXPMJ3NYDM32BRHA8YH (min time: 1697450400000, max time: 1697457600000) 01HCW9ARN3G0EJMAS4HB3Z8W3F (min time: 1697450400000, max time: 1697457600000) 01HCW9DF4ZGG25BRS09B7F74X9 (min time: 1697450400000, max time: 1697457600000) 01HCW9ANK1TBBD3NDEYW55MEAD (min time: 1697450400000, max time: 1697457600000) 01HCW94VNCB1A1M8S1FVCZWH65 (min time: 1697450400000, max time: 1697457600000) 01HCW9ABKN7SD08GW53CT8B4Q5 (min time: 1697450400000, max time: 1697457600000) 01HCW9A7P4NPB9F615KBD6TGZY (min time: 1697450400000, max time: 1697457600000) 01HCW9ANSN8858929MX4NP5839 (min time: 1697450400000, max time: 1697457600000) 01HCW9AWTXMJC433BS4S2EFMJB (min time: 1697450400000, max time: 1697457600000) 01HCW9AQD3T052P8BWNAMCCQ2D (min time: 1697450400000, max time: 1697457600000) 01HCW97R6N93ERRHQ1XFE06Q5E (min time: 1697450400000, max time: 1697457600000) 01HCW9D4BQRM32S1F6XAZ2C0QZ (min time: 1697450400000, max time: 1697457600000) 01HCW99NCXR8BEWJ04MJB4JRSF (min time: 1697450400000, max time: 1697457600000) 01HCW95E83727PJC00DP7WXX3T (min time: 1697450400000, max time: 1697457600000) 01HCW9AR1S71JZD4B79M1ADYYE (min time: 1697450400000, max time: 1697457600000) 01HCW94Y269VJ81V1GM37V1V3E (min time: 1697450400000, max time: 1697457600000) 01HCW954EP3QMMD0H0QYCAN2X7 (min time: 1697450400000, max time: 1697457600000) 01HCW95G5TXN2T60RWCG3NWXBH (min time: 1697450400000, max time: 1697457600000) 01HCW9AW5JVG0J20YW7037YSCF (min time: 1697450400000, max time: 1697457600000) 01HCW7J5V1Y4AES2VXQZ67JHW7 (min time: 1697450400000, max time: 1697457600000) 01HCW9AGP7AFBXPV0QG1Z9DAJN (min time: 1697450400000, max time: 1697457600000) 01HCW9APYVBEFJP9JBAM2WA4M0 (min time: 1697450400001, max time: 1697457600000) 01HCW957JWMX807P9W0P7B9YY7 (min time: 1697450400001, max time: 1697457600000) 01HCW9CNHZEVJS43N82XJXEP4J (min time: 1697450400001, max time: 1697457600000) 01HCW9D58PM10G7RGAK70CZ6A1 (min time: 1697450400001, max time: 1697457600000) 01HCW950D8AKTNBZGM9GN2HZM7 (min time: 1697450400001, max time: 1697457600000) 01HCW98QZ5Q9SMNZEZCFNJ4XN8 (min time: 1697450400001, max time: 1697457600000) 01HCW95AZ2R9CX4XYT88B1QHXS (min time: 1697450400001, max time: 1697457600000) 01HCW7XY0ZESYAGQVJ3MKDMQKX (min time: 1697450400001, max time: 1697457600000) 01HCW9AVRCS1CZW1VXBMPX7803 (min time: 1697450400001, max time: 1697457600000) 01HCW9DD3FW6KQ841P27KJK5D8 (min time: 1697450400001, max time: 1697457600000) 01HCW9BWNJ9Z98DCD42CN92VRZ (min time: 1697450400001, max time: 1697457600000) 01HCW99Y71N7DHP38XP2FWZBE4 (min time: 1697450400001, max time: 1697457600000) 01HCW9DMC30C8ZVA16NHK4S2RB (min time: 1697450400001, max time: 1697457600000) 01HCW9AWVA3VJXG8CP6SPS1GT8 (min time: 1697450400001, max time: 1697457600000) 01HCW7P0P6KPJN24KZ6CS69XA2 (min time: 1697450400001, max time: 1697457600000) 01HCW95ASC3N3PQ50SFF999XXS (min time: 1697450400001, max time: 1697457600000) 01HCW9AMJDK4FVFCSYA961S9DW (min time: 1697450400001, max time: 1697457600000) 01HCW953R3T6059ZGWHHXXGMWB (min time: 1697450400001, max time: 1697457600000) 01HCW955ZYPHMM71EXS4CCCNPK (min time: 1697450400001, max time: 1697457600000) 01HCW8AFZW1JHJDT89ZQQ2H9G6 (min time: 1697450400001, max time: 1697457600000) 01HCW94YTXY53B5YCNAMXAHDSQ (min time: 1697450400001, max time: 1697457600000) 01HCW9B5D7BPFY06YJSBN9B5KA (min time: 1697450400001, max time: 1697457600000) 01HCW9DSZXTMS823J18NH66AQM (min time: 1697450400001, max time: 1697457600000) 01HCW99FXNGW1C9WS6WBQDCC3K (min time: 1697450400002, max time: 1697457600000) 01HCW9D43HHR6GA4MARC1Z7J2M (min time: 1697450400002, max time: 1697457600000) 01HCW9AH0HYD58VTWGG4S5EX99 (min time: 1697450400002, max time: 1697457600000) 01HCW9AMNZ5C7X5GB1EXG6NJ6R (min time: 1697450400002, max time: 1697457600000) 01HCW98FVQ9RHCBJ9ZBYPSMSSV (min time: 1697450400002, max time: 1697457600000) 01HCW9CAWMRX7YC91BMS59A12E (min time: 1697450400002, max time: 1697457600000) 01HCW9B4BET2CDAKS0A09Q9AJ8 (min time: 1697450400002, max time: 1697457600000) 01HCW96GKHN26HVQHJZ1HQEA3P (min time: 1697450400003, max time: 1697457600000) 01HCW9DE1MZXD7G5ZD3HVNSJ47 (min time: 1697450400003, max time: 1697457600000) 01HCW843AWYT8HQG1VGAW35FQK (min time: 1697450400003, max time: 1697457600000) 01HCW9BREG7X0VT5G1TGYK58QA (min time: 1697450400003, max time: 1697457600000) 01HCW9ARM74A21RQ5BKYXE01YS (min time: 1697450400004, max time: 1697457600000) 01HCW9BRFJV2C94ZJCCJ7HXWT5 (min time: 1697450400004, max time: 1697457600000) 01HCW97ATEXAANENGD472D0EBB (min time: 1697450400004, max time: 1697457600000) 01HCW9547581FW2P1PHTFQ4W9K (min time: 1697450400004, max time: 1697457600000) 01HCW9CVH17X1TTCPJ245XKVQD (min time: 1697450400004, max time: 1697457600000) 01HCW95528PDASMJ5MKVGYHCFQ (min time: 1697450400004, max time: 1697457600000) 01HCW9B5CAD1K9VR0H2EH5RR96 (min time: 1697450400006, max time: 1697457600000) 01HCW9AH4HQBTQM9CMGBMDC8Y8 (min time: 1697450400006, max time: 1697457600000) 01HCW9BWP8AWSFN8MH4P3SYR3C (min time: 1697450400006, max time: 1697457600000) 01HCW95T2VK2CXR65BTQD3XYR2 (min time: 1697450400006, max time: 1697457600000) 01HCW9DVMVAFPXDV4N06GJG4WP (min time: 1697450400006, max time: 1697457600000) 01HCW950VCN1ZBW4TVH30CEPVD (min time: 1697450400006, max time: 1697457600000) 01HCW9B3SAKJEQ1DZRE93MBZCF (min time: 1697450400006, max time: 1697457600000) 01HCW9BB6SX02BF8QE5CT2QBTH (min time: 1697450400006, max time: 1697457600000) 01HCW959YP7JRHPT2ZT3GBYZ3V (min time: 1697450400009, max time: 1697457600000) 01HCW9DRNX544RNFF29C68FT4M (min time: 1697450400009, max time: 1697457600000) 01HCW96AB3CSZ2K77EQMGSAHD6 (min time: 1697450400010, max time: 1697457600000) 01HCWA6VQ5698J86B2NSQVV9WW (min time: 1697452876670, max time: 1697457600000) 01HCWA1RR752S54GJJZX2ZNRKR (min time: 1697452884390, max time: 1697457600000) 01HCW9JHHTZQPD436C8E5WMJFC (min time: 1697452886569, max time: 1697457600000) 01HCWAZM8G2RRKZTEGQ8WG6T0Y (min time: 1697452886758, max time: 1697457600000) 01HCWAHE4K84ZE3S7NXN1E3SHK (min time: 1697452886872, max time: 1697457600000) 01HCW9Q7CTFV22CCYBY8EPETEG (min time: 1697452886972, max time: 1697457600000) 01HCW9WKR852BHJ0ZYQ5790RFZ (min time: 1697452888899, max time: 1697457600000) 01HCWAB5J0897T9VJ26SA36JEV (min time: 1697452888899, max time: 1697457600000) 01HCWAV8FH3R00RQ744VXKEBG9 (min time: 1697452891793, max time: 1697457600000) 01HCWAPP4P47H6GJTE57BQHZ00 (min time: 1697452891793, max time: 1697457600000) 01HCW8DWXHTH4X6WEGJMAMWK5A (min time: 1697452997619, max time: 1697457600000) 01HCW90AKHZENW3V8TBMP9BNB0 (min time: 1697453001867, max time: 1697457600000) 01HCW7TTM6RV53EZ3SSMQ911JK (min time: 1697453060709, max time: 1697457600000) 01HCW7TTM06F8T792ZNGD076EY (min time: 1697453060709, max time: 1697457600000) 01HCWB59HYXASVYR7MH5PJZKCC (min time: 1697456120833, max time: 1697457600000) 01HCWBBHSAXY9D96NQWZATP0A6 (min time: 1697456463481, max time: 1697457600000) 01HCWBKC1F9GC9CMM2YMCPQC6E (min time: 1697456605750, max time: 1697457600000) 01HCWBSSFF9473CPJRGKCR2P2X (min time: 1697456760770, max time: 1697457600000) 01HCWBZWPT5WGKNBZHZARDA3GT (min time: 1697456929949, max time: 1697457600000) 01HCWC7KWQB4MPXM4VTZ9RA6AG (min time: 1697457021954, max time: 1697457600000) 01HCWCDR19J0HG9NG3QPB63Y7Y (min time: 1697457299399, max time: 1697457600000) 01HCWCKGTA82F9CBRYRFKXPQAR (min time: 1697457470943, max time: 1697457600000)]","ts":"2023-10-17T08:04:50.43689535Z"}{"caller":"objstore.go:361","component":"compactor","file":"01HCW9AQXPMJ3NYDM32BRHA8YH/meta.json","group":"0@{__org_id__=\"GC\"}","groupKey":"0@7253914978157373696","level":"debug","msg":"not downloading again because a provided path matches this one","org_id":"GC","ts":"2023-10-17T08:04:50.449742708Z"}{"caller":"objstore.go:361","component":"compactor","file":"01HCW98DQ8Z3A1VF9ZF71SYJZX/meta.json","group":"0@{__org_id__=\"GC\"}","groupKey":"0@7253914978157373696","level":"debug","msg":"not downloading again because a provided path matches this one","org_id":"GC","ts":"2023-10-17T08:04:50.453705345Z"}{"caller":"objstore.go:361","component":"compactor","file":"01HCW6SYRH47K26KZS2H3K3H18/meta.json","group":"0@{__org_id__=\"GC\"}","groupKey":"0@7253914978157373696","level":"debug","msg":"not downloading again because a provided path matches this one","org_id":"GC","ts":"2023-10-17T08:04:50.453822594Z"}unexpected fault address 0x7ffef52a17ad
[signal SIGBUS: bus error code=0x2 addr=0x7ffef52a17ad pc=0x9bca58]
goroutine 1792576 [running]:
runtime.throw({0x2863df0?, 0xc001b08fe0?}) /usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0xc001b08f70 sp=0xc001b08f40 pc=0x43907d
/usr/local/go/src/runtime/signal_unix.go:834 +0x125 fp=0xc001b08fd0 sp=0xc001b08f70 pc=0x44fde5
github.com/dennwc/varint.Uvarint({0x7ffef52a17ad?, 0xc001b09010?, 0x455339?}) /__w/cortex/cortex/vendor/github.com/dennwc/varint/varint.go:75 +0x18 fp=0xc001b08fd8 sp=0xc001b08fd0 pc=0x9bca58
github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).Uvarint64(0xc001b09060)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go:242 +0x3e fp=0xc001b09000 sp=0xc001b08fd8 pc=0xce817e
github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).UvarintBytes(0xc001b09060)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go:206 +0x25 fp=0xc001b09020 sp=0xc001b09000 pc=0xce7f25
github.com/prometheus/prometheus/tsdb/index.Symbols.Lookup({{0x2e8a720, 0xc001cda000}, 0x2, 0x5, {0xc005280000, 0x14a50, 0x14a50}, 0x2949f1}, 0x1018) /__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/index/index.go:1316 +0x245 fp=0xc001b09098 sp=0xc001b09020 pc=0xcf29c5
github.com/prometheus/prometheus/tsdb/index.(*Reader).lookupSymbol(0xc00062e360, 0x428f25?)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/index/index.go:1447 +0xd8 fp=0xc001b09138 sp=0xc001b09098 pc=0xcf3818
github.com/prometheus/prometheus/tsdb/index.(*Reader).lookupSymbol-fm(0x1b09210?)
<autogenerated>:1 +0x2b fp=0xc001b09158 sp=0xc001b09138 pc=0xcfdceb
github.com/prometheus/prometheus/tsdb/index.(*Decoder).Series(0xc00102e000, {0x7fff2ef9c7c2?, 0x39d437c0?, 0x7ffff7fbd108?}, 0xc001b09570, 0xc001b09558) /__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/index/index.go:1861 +0x151 fp=0xc001b092c8 sp=0xc001b09158 pc=0xcf63d1
github.com/prometheus/prometheus/tsdb/index.(*Reader).Series(0xc00062e360, 0x39d437c, 0xc001b09570?, 0xc00364b380?)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/index/index.go:1611 +0x127 fp=0xc001b09378 sp=0xc001b092c8 pc=0xcf49c7
github.com/thanos-io/thanos/pkg/block.GatherIndexHealthStats({0x2e7a3a0, 0xc00013d770}, {_, _}, _, _) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/index.go:255 +0x566 fp=0xc001b09660 sp=0xc001b09378 pc=0x1972306
github.com/thanos-io/thanos/pkg/compact.(*Group).compact.func2.1.2({0x2e95968?, 0xc00013c140?}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1037 +0xdd fp=0xc001b09978 sp=0xc001b09660 pc=0x1dd26fd
github.com/thanos-io/thanos/pkg/tracing.DoInSpanWithErr({0x2e95968?, 0xc00013c140?}, {0x28a1f68?, 0x8?}, 0xc001b09f38, {0xc0013d8020?, 0xc003443b00?, 0x416050?}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:82 +0xd0 fp=0xc001b09a18 sp=0xc001b09978 pc=0x1ccad10
github.com/thanos-io/thanos/pkg/compact.(*Group).compact.func2.1()
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1036 +0x45c fp=0xc001b09f78 sp=0xc001b09a18 pc=0x1dd203c
golang.org/x/sync/errgroup.(*Group).Go.func1()
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64 fp=0xc001b09fe0 sp=0xc001b09f78 pc=0xd012c4
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001b09fe8 sp=0xc001b09fe0 pc=0x46fd61
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 1 [select, 810 minutes]:
runtime.gopark(0xc000dbc068?, 0x2?, 0xe8?, 0xa4?, 0xc000dbc064?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0017fbef0 sp=0xc0017fbed0 pc=0x43bd96
runtime.selectgo(0xc0017fc068, 0xc000dbc060, 0x16374e0?, 0x0, 0x2e959a0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0017fc030 sp=0xc0017fbef0 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AwaitStopped(0xc001262720, {0x2e959a0, 0xc00007a028}) /__w/cortex/cortex/pkg/util/services/manager.go:145 +0x6d fp=0xc0017fc098 sp=0xc0017fc030 pc=0x1639dcd
github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run(0xc000e60000)
/__w/cortex/cortex/pkg/cortex/cortex.go:459 +0x925 fp=0xc0017fc260 sp=0xc0017fc098 pc=0x2140145
/__w/cortex/cortex/cmd/cortex/main.go:196 +0xdf0 fp=0xc0017fff80 sp=0xc0017fc260 pc=0x214d710
/usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc0017fffe0 sp=0xc0017fff80 pc=0x43b967
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0017fffe8 sp=0xc0017fffe0 pc=0x46fd61
goroutine 2 [force gc (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000110fb0 sp=0xc000110f90 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
/usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc000110fe0 sp=0xc000110fb0 pc=0x43bbd0
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000110fe8 sp=0xc000110fe0 pc=0x46fd61
created by runtime.init.6
/usr/local/go/src/runtime/proc.go:293 +0x25
goroutine 3 [GC sweep wait]:
runtime.gopark(0x42c5f01?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000111780 sp=0xc000111760 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
/usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc0001117c8 sp=0xc000111780 pc=0x425e3e
/usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc0001117e0 sp=0xc0001117c8 pc=0x41b0a6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001117e8 sp=0xc0001117e0 pc=0x46fd61
created by runtime.gcenable
/usr/local/go/src/runtime/mgc.go:178 +0x6b
runtime.gopark(0xeee7a00db384e?, 0x92bdc?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000111f70 sp=0xc000111f50 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x42ca6a0)
/usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc000111fa0 sp=0xc000111f70 pc=0x423d13
/usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc000111fc8 sp=0xc000111fa0 pc=0x424305
/usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc000111fe0 sp=0xc000111fc8 pc=0x41b046
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000111fe8 sp=0xc000111fe0 pc=0x46fd61
created by runtime.gcenable
/usr/local/go/src/runtime/mgc.go:179 +0xaa
goroutine 5 [finalizer wait, 810 minutes]:
runtime.gopark(0x43c112?, 0x43bb05?, 0x0?, 0x0?, 0xc000110770?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000110628 sp=0xc000110608 pc=0x43bd96
/usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc0001107e0 sp=0xc000110628 pc=0x41a0e7
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001107e8 sp=0xc0001107e0 pc=0x46fd61
created by runtime.createfing
/usr/local/go/src/runtime/mfinal.go:163 +0x45
goroutine 12 [GC worker (idle), 810 minutes]:
runtime.gopark(0xc000132660?, 0xc000112788?, 0x3d?, 0x5d?, 0xc0001127a8?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000112750 sp=0xc000112730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001127e0 sp=0xc000112750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001127e8 sp=0xc0001127e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 18 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010c750 sp=0xc00010c730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010c7e0 sp=0xc00010c750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010c7e8 sp=0xc00010c7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 19 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010cf50 sp=0xc00010cf30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010cfe0 sp=0xc00010cf50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010cfe8 sp=0xc00010cfe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 20 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010d750 sp=0xc00010d730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010d7e0 sp=0xc00010d750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010d7e8 sp=0xc00010d7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 13 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000112f50 sp=0xc000112f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000112fe0 sp=0xc000112f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000112fe8 sp=0xc000112fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 14 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000113750 sp=0xc000113730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001137e0 sp=0xc000113750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001137e8 sp=0xc0001137e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 15 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000113f50 sp=0xc000113f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000113fe0 sp=0xc000113f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000113fe8 sp=0xc000113fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 34 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000312750 sp=0xc000312730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0003127e0 sp=0xc000312750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0003127e8 sp=0xc0003127e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 21 [GC worker (idle), 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010df50 sp=0xc00010df30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010dfe0 sp=0xc00010df50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010dfe8 sp=0xc00010dfe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 16 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x3?, 0x8e?, 0xbd?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00030e750 sp=0xc00030e730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00030e7e0 sp=0xc00030e750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00030e7e8 sp=0xc00030e7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 49 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0xa9?, 0xe7?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00030ef50 sp=0xc00030ef30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00030efe0 sp=0xc00030ef50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00030efe8 sp=0xc00030efe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 50 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x41?, 0xc6?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00030f750 sp=0xc00030f730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00030f7e0 sp=0xc00030f750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00030f7e8 sp=0xc00030f7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 35 [GC worker (idle)]:
runtime.gopark(0xeee7a00d10847?, 0x1?, 0x78?, 0x8e?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000312f50 sp=0xc000312f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000312fe0 sp=0xc000312f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000312fe8 sp=0xc000312fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 22 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x50?, 0x7d?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010e750 sp=0xc00010e730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010e7e0 sp=0xc00010e750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010e7e8 sp=0xc00010e7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 51 [GC worker (idle)]:
runtime.gopark(0xeee7a00cb7cc8?, 0x3?, 0x9e?, 0x9d?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00030ff50 sp=0xc00030ff30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00030ffe0 sp=0xc00030ff50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00030ffe8 sp=0xc00030ffe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 36 [GC worker (idle)]:
runtime.gopark(0xeee79a641f156?, 0x1?, 0xae?, 0x38?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000313750 sp=0xc000313730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0003137e0 sp=0xc000313750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0003137e8 sp=0xc0003137e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 52 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x91?, 0x2e?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000310750 sp=0xc000310730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0003107e0 sp=0xc000310750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0003107e8 sp=0xc0003107e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 37 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x35?, 0x5e?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000313f50 sp=0xc000313f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000313fe0 sp=0xc000313f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000313fe8 sp=0xc000313fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 53 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x20?, 0x28?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000310f50 sp=0xc000310f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000310fe0 sp=0xc000310f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000310fe8 sp=0xc000310fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 23 [GC worker (idle)]:
runtime.gopark(0xeee7a00cfcaf6?, 0x3?, 0xbc?, 0xdb?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010ef50 sp=0xc00010ef30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010efe0 sp=0xc00010ef50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010efe8 sp=0xc00010efe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 54 [GC worker (idle)]:
runtime.gopark(0xeee7a00d0fe4d?, 0x3?, 0x70?, 0x6e?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000311750 sp=0xc000311730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0003117e0 sp=0xc000311750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0003117e8 sp=0xc0003117e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 24 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x66?, 0xc6?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010f750 sp=0xc00010f730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010f7e0 sp=0xc00010f750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010f7e8 sp=0xc00010f7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 38 [GC worker (idle)]:
runtime.gopark(0xeee7a00d0fb3b?, 0x3?, 0xc?, 0x94?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000314750 sp=0xc000314730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0003147e0 sp=0xc000314750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0003147e8 sp=0xc0003147e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 39 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x3?, 0xfc?, 0x35?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000314f50 sp=0xc000314f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000314fe0 sp=0xc000314f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000314fe8 sp=0xc000314fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 55 [GC worker (idle), 72 minutes]:
runtime.gopark(0x42fd960?, 0x3?, 0xc7?, 0xf1?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000311f50 sp=0xc000311f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000311fe0 sp=0xc000311f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000311fe8 sp=0xc000311fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 40 [GC worker (idle)]:
runtime.gopark(0xeee7a00d0f82e?, 0x1?, 0x9e?, 0xda?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000315750 sp=0xc000315730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0003157e0 sp=0xc000315750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0003157e8 sp=0xc0003157e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 56 [GC worker (idle)]:
runtime.gopark(0xeee7a00d10113?, 0x3?, 0xab?, 0x5e?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000528750 sp=0xc000528730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005287e0 sp=0xc000528750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005287e8 sp=0xc0005287e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 41 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x69?, 0xda?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000315f50 sp=0xc000315f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000315fe0 sp=0xc000315f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000315fe8 sp=0xc000315fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 57 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x2c?, 0xfd?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000528f50 sp=0xc000528f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000528fe0 sp=0xc000528f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000528fe8 sp=0xc000528fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 42 [GC worker (idle), 69 minutes]:
runtime.gopark(0xeeab3311d038b?, 0x1?, 0x3?, 0xf4?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000524750 sp=0xc000524730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005247e0 sp=0xc000524750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005247e8 sp=0xc0005247e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 25 [GC worker (idle)]:
runtime.gopark(0xeee79a63ff1aa?, 0x3?, 0xf1?, 0xb1?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010ff50 sp=0xc00010ff30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00010ffe0 sp=0xc00010ff50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010ffe8 sp=0xc00010ffe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 26 [GC worker (idle)]:
runtime.gopark(0xeee79a63def15?, 0x3?, 0xb0?, 0x39?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001ba750 sp=0xc0001ba730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001ba7e0 sp=0xc0001ba750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001ba7e8 sp=0xc0001ba7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 58 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0xc8?, 0xed?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000529750 sp=0xc000529730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005297e0 sp=0xc000529750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005297e8 sp=0xc0005297e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 43 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x50?, 0xe4?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000524f50 sp=0xc000524f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000524fe0 sp=0xc000524f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000524fe8 sp=0xc000524fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 59 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0xf1?, 0x46?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000529f50 sp=0xc000529f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000529fe0 sp=0xc000529f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000529fe8 sp=0xc000529fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 60 [GC worker (idle)]:
runtime.gopark(0xeee7a00c57d7d?, 0x3?, 0x64?, 0xd?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00052a750 sp=0xc00052a730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00052a7e0 sp=0xc00052a750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00052a7e8 sp=0xc00052a7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 27 [GC worker (idle)]:
runtime.gopark(0xeee7a00d10a19?, 0x3?, 0x4c?, 0x27?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001baf50 sp=0xc0001baf30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001bafe0 sp=0xc0001baf50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001bafe8 sp=0xc0001bafe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 28 [GC worker (idle)]:
runtime.gopark(0xeee79a63f1b84?, 0x3?, 0x15?, 0xef?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001bb750 sp=0xc0001bb730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001bb7e0 sp=0xc0001bb750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001bb7e8 sp=0xc0001bb7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 44 [GC worker (idle)]:
runtime.gopark(0xeee78cb29b0d5?, 0x3?, 0x3b?, 0x73?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000525750 sp=0xc000525730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005257e0 sp=0xc000525750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005257e8 sp=0xc0005257e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 29 [GC worker (idle)]:
runtime.gopark(0xeee79a641e8bf?, 0x3?, 0x98?, 0x2b?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001bbf50 sp=0xc0001bbf30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001bbfe0 sp=0xc0001bbf50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001bbfe8 sp=0xc0001bbfe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 45 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x7?, 0xd6?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000525f50 sp=0xc000525f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000525fe0 sp=0xc000525f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000525fe8 sp=0xc000525fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 46 [GC worker (idle)]:
runtime.gopark(0xeee7a00d0f8d3?, 0x3?, 0x10?, 0x2a?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000526750 sp=0xc000526730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005267e0 sp=0xc000526750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005267e8 sp=0xc0005267e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 47 [GC worker (idle)]:
runtime.gopark(0xeee7a00c96752?, 0x1?, 0x44?, 0x19?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000526f50 sp=0xc000526f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000526fe0 sp=0xc000526f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000526fe8 sp=0xc000526fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 48 [GC worker (idle)]:
runtime.gopark(0xeee79a63df48d?, 0x3?, 0x55?, 0xcc?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000527750 sp=0xc000527730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005277e0 sp=0xc000527750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005277e8 sp=0xc0005277e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 65 [GC worker (idle), 72 minutes]:
runtime.gopark(0x42fd960?, 0x3?, 0xf6?, 0xb?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000527f50 sp=0xc000527f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000527fe0 sp=0xc000527f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000527fe8 sp=0xc000527fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 66 [GC worker (idle)]:
runtime.gopark(0xeee79d4b9cb16?, 0x3?, 0x8b?, 0x39?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001b6750 sp=0xc0001b6730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001b67e0 sp=0xc0001b6750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001b67e8 sp=0xc0001b67e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 61 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0xd5?, 0x32?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00052af50 sp=0xc00052af30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00052afe0 sp=0xc00052af50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00052afe8 sp=0xc00052afe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 30 [GC worker (idle)]:
runtime.gopark(0xeee79a641f120?, 0x3?, 0xc0?, 0x23?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001bc750 sp=0xc0001bc730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001bc7e0 sp=0xc0001bc750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001bc7e8 sp=0xc0001bc7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 67 [GC worker (idle), 68 minutes]:
runtime.gopark(0xeeac31e14ba48?, 0x3?, 0x58?, 0x58?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001b6f50 sp=0xc0001b6f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001b6fe0 sp=0xc0001b6f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001b6fe8 sp=0xc0001b6fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 62 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0x6a?, 0xf?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00052b750 sp=0xc00052b730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00052b7e0 sp=0xc00052b750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00052b7e8 sp=0xc00052b7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 68 [GC worker (idle)]:
runtime.gopark(0xeee7a00d1041c?, 0x1?, 0x48?, 0x4d?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001b7750 sp=0xc0001b7730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001b77e0 sp=0xc0001b7750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001b77e8 sp=0xc0001b77e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 31 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0xb7?, 0xcf?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001bcf50 sp=0xc0001bcf30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001bcfe0 sp=0xc0001bcf50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001bcfe8 sp=0xc0001bcfe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 63 [GC worker (idle)]:
runtime.gopark(0xeee7a00cafda6?, 0x3?, 0x24?, 0xa7?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00052bf50 sp=0xc00052bf30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00052bfe0 sp=0xc00052bf50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00052bfe8 sp=0xc00052bfe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 81 [GC worker (idle)]:
runtime.gopark(0xeee7a00d0f533?, 0x3?, 0xd4?, 0xf8?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000586750 sp=0xc000586730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0005867e0 sp=0xc000586750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005867e8 sp=0xc0005867e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 69 [GC worker (idle)]:
runtime.gopark(0xeee7a00d0ffc6?, 0x3?, 0xf7?, 0xc6?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001b7f50 sp=0xc0001b7f30 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001b7fe0 sp=0xc0001b7f50 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001b7fe8 sp=0xc0001b7fe0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
goroutine 32 [GC worker (idle)]:
runtime.gopark(0x42fd960?, 0x1?, 0xe1?, 0x13?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001bd750 sp=0xc0001bd730 pc=0x43bd96
/usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001bd7e0 sp=0xc0001bd750 pc=0x41ce11
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001bd7e8 sp=0xc0001bd7e0 pc=0x46fd61
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1199 +0x25
runtime.gopark(0xc000588f88?, 0x3?, 0x9?, 0x18?, 0xc000588f72?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000588df8 sp=0xc000588dd8 pc=0x43bd96
runtime.selectgo(0xc000588f88, 0xc000588f6c, 0x0?, 0x0, 0x3?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000588f38 sp=0xc000588df8 pc=0x44c03e
go.opencensus.io/stats/view.(*worker).start(0xc000300800)
/__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:292 +0xad fp=0xc000588fc8 sp=0xc000588f38 pc=0x1406d4d
go.opencensus.io/stats/view.init.0.func1()
/__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:34 +0x26 fp=0xc000588fe0 sp=0xc000588fc8 pc=0x1405f86
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000588fe8 sp=0xc000588fe0 pc=0x46fd61
created by go.opencensus.io/stats/view.init.0
/__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:34 +0x96
goroutine 122 [chan receive, 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0005886f8 sp=0xc0005886d8 pc=0x43bd96
runtime.chanrecv(0xc000292240, 0xc0005887c8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000588788 sp=0xc0005886f8 pc=0x406f9d
runtime.chanrecv2(0x0?, 0x0?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc0005887b0 sp=0xc000588788 pc=0x406ad8
github.com/cortexproject/cortex/pkg/alertmanager.init.0.func1()
/__w/cortex/cortex/pkg/alertmanager/alertmanager.go:128 +0x45 fp=0xc0005887e0 sp=0xc0005887b0 pc=0x1912fa5
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005887e8 sp=0xc0005887e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/alertmanager.init.0
/__w/cortex/cortex/pkg/alertmanager/alertmanager.go:124 +0x25
runtime.gopark(0xc002563488?, 0x4?, 0xc5?, 0x31?, 0xc002563330?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc002563188 sp=0xc002563168 pc=0x43bd96
runtime.selectgo(0xc002563488, 0xc002563328, 0xc00126a060?, 0x0, 0x0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0025632c8 sp=0xc002563188 pc=0x44c03e
net/http.(*Transport).getConn(0xc00059b400, 0xc001e0c100, {{}, 0x0, {0xc00309a000, 0x4}, {0xc000750080, 0x37}, 0x0}) /usr/local/go/src/net/http/transport.go:1382 +0x5c6 fp=0xc0025634e0 sp=0xc0025632c8 pc=0x8209a6
net/http.(*Transport).roundTrip(0xc00059b400, 0xc00309e000)
/usr/local/go/src/net/http/transport.go:590 +0x79e fp=0xc002563708 sp=0xc0025634e0 pc=0x81c89e
net/http.(*Transport).RoundTrip(0xc001884540?, 0x2e7f6e0?)
/usr/local/go/src/net/http/roundtrip.go:17 +0x19 fp=0xc002563728 sp=0xc002563708 pc=0x803359
net/http.send(0xc00309e000, {0x2e7f6e0, 0xc00059b400}, {0x8?, 0x27f4fc0?, 0x0?}) /usr/local/go/src/net/http/client.go:252 +0x5f7 fp=0xc002563920 sp=0xc002563728 pc=0x7bf2f7
net/http.(*Client).send(0xc00034eed0, 0xc00309e000, {0x42fd960?, 0x40d4be?, 0x0?}) /usr/local/go/src/net/http/client.go:176 +0x9b fp=0xc002563998 sp=0xc002563920 pc=0x7beb7b
net/http.(*Client).do(0xc00034eed0, 0xc00309e000)
/usr/local/go/src/net/http/client.go:716 +0x8fb fp=0xc002563ba0 sp=0xc002563998 pc=0x7c0e7b
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:582
net/http.(*Client).Get(0xc000fa9cc8?, {0xc00309a000?, 0x2868940?}) /usr/local/go/src/net/http/client.go:480 +0x6a fp=0xc002563bf8 sp=0xc002563ba0 pc=0x7c040a
github.com/uber/jaeger-client-go.(*httpSamplingStrategyFetcher).Fetch(0xc00034eeb0, {0xc00007a210, 0x10}) /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:325 +0x1c5 fp=0xc002563e60 sp=0xc002563bf8 pc=0x10e2165
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).UpdateSampler(0xc001112750)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:178 +0x62 fp=0xc002563f18 sp=0xc002563e60 pc=0x10e1382
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollControllerWithTicker(0xc001112750, 0xc000f1cb40)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:155 +0x2e fp=0xc002563f88 sp=0xc002563f18 pc=0x10e110e
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollController(0xc001112750)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:148 +0x6d fp=0xc002563fc8 sp=0xc002563f88 pc=0x10e102d
github.com/uber/jaeger-client-go.NewRemotelyControlledSampler.func1()
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:87 +0x26 fp=0xc002563fe0 sp=0xc002563fc8 pc=0x10e0726
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc002563fe8 sp=0xc002563fe0 pc=0x46fd61
created by github.com/uber/jaeger-client-go.NewRemotelyControlledSampler
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:87 +0x15b
runtime.gopark(0xc0001c5f88?, 0x2?, 0x3a?, 0x95?, 0xc0001c5f6c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001c5df0 sp=0xc0001c5dd0 pc=0x43bd96
runtime.selectgo(0xc0001c5f88, 0xc0001c5f68, 0x46cb59?, 0x0, 0xc000584790?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0001c5f30 sp=0xc0001c5df0 pc=0x44c03e
github.com/uber/jaeger-client-go/utils.(*reconnectingUDPConn).reconnectLoop(0xc0008a6e00, 0x1?)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:70 +0xbc fp=0xc0001c5fc0 sp=0xc0001c5f30 pc=0x10cda9c
github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn.func1()
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x2a fp=0xc0001c5fe0 sp=0xc0001c5fc0 pc=0x10cd9aa
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001c5fe8 sp=0xc0001c5fe0 pc=0x46fd61
created by github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x205
goroutine 249 [chan receive, 810 minutes]:
runtime.gopark(0xc0009be0c0?, 0x3?, 0x48?, 0xef?, 0xc001257008?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2eee8 sp=0xc000e2eec8 pc=0x43bd96
runtime.chanrecv(0xc0012627e0, 0xc000e2efc8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e2ef78 sp=0xc000e2eee8 pc=0x406f9d
runtime.chanrecv2(0x2e98ee0?, 0xc000e09e90?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e2efa0 sp=0xc000e2ef78 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000e2efe0 sp=0xc000e2efa0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2efe8 sp=0xc000e2efe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 169 [syscall, 810 minutes]:
runtime.notetsleepg(0x0?, 0x0?)
/usr/local/go/src/runtime/lock_futex.go:236 +0x34 fp=0xc000586fa0 sp=0xc000586f68 pc=0x40c734
/usr/local/go/src/runtime/sigqueue.go:152 +0x2f fp=0xc000586fc0 sp=0xc000586fa0 pc=0x46c08f
/usr/local/go/src/os/signal/signal_unix.go:23 +0x19 fp=0xc000586fe0 sp=0xc000586fc0 pc=0x111f899
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000586fe8 sp=0xc000586fe0 pc=0x46fd61
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x2a
goroutine 344 [chan receive, 809 minutes]:
runtime.gopark(0xc000f0a000?, 0xc000583fc8?, 0x80?, 0x68?, 0xc000583fa8?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000583ee8 sp=0xc000583ec8 pc=0x43bd96
runtime.chanrecv(0xc000a4c6c0, 0xc000583fc8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000583f78 sp=0xc000583ee8 pc=0x406f9d
runtime.chanrecv2(0x2e98ea0?, 0xc000cc5ad0?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000583fa0 sp=0xc000583f78 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000583fe0 sp=0xc000583fa0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000583fe8 sp=0xc000583fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 152 [runnable]:
runtime.gopark(0xc000e2df78?, 0x2?, 0x9?, 0x18?, 0xc000e2df14?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2dd80 sp=0xc000e2dd60 pc=0x43bd96
runtime.selectgo(0xc000e2df78, 0xc000e2df10, 0x0?, 0x0, 0x2?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000e2dec0 sp=0xc000e2dd80 pc=0x44c03e
github.com/uber/jaeger-client-go.(*remoteReporter).processQueue(0xc00035a2a0)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:296 +0xde fp=0xc000e2dfc8 sp=0xc000e2dec0 pc=0x10ddbde
github.com/uber/jaeger-client-go.NewRemoteReporter.func1()
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x26 fp=0xc000e2dfe0 sp=0xc000e2dfc8 pc=0x10dd866
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2dfe8 sp=0xc000e2dfe0 pc=0x46fd61
created by github.com/uber/jaeger-client-go.NewRemoteReporter
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x250
goroutine 248 [chan receive, 808 minutes]:
runtime.gopark(0xc0009be120?, 0x3?, 0x80?, 0xc2?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2e6e8 sp=0xc000e2e6c8 pc=0x43bd96
runtime.chanrecv(0xc001262780, 0xc000e2e7c8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e2e778 sp=0xc000e2e6e8 pc=0x406f9d
runtime.chanrecv2(0x2e98ee0?, 0xc000e09e78?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e2e7a0 sp=0xc000e2e778 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000e2e7e0 sp=0xc000e2e7a0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2e7e8 sp=0xc000e2e7e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 250 [chan receive, 810 minutes]:
runtime.gopark(0xc0009be180?, 0x3?, 0x0?, 0x0?, 0xc001257008?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2f6e8 sp=0xc000e2f6c8 pc=0x43bd96
runtime.chanrecv(0xc001262840, 0xc000e2f7c8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e2f778 sp=0xc000e2f6e8 pc=0x406f9d
runtime.chanrecv2(0x2e98ee0?, 0xc000e09ea8?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e2f7a0 sp=0xc000e2f778 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000e2f7e0 sp=0xc000e2f7a0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2f7e8 sp=0xc000e2f7e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 251 [chan receive, 810 minutes]:
runtime.gopark(0xc000a4c540?, 0xc000e2ffc8?, 0x80?, 0x9e?, 0xc001257008?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2fee8 sp=0xc000e2fec8 pc=0x43bd96
runtime.chanrecv(0xc0012628a0, 0xc000e2ffc8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e2ff78 sp=0xc000e2fee8 pc=0x406f9d
runtime.chanrecv2(0x2e98ee0?, 0xc000e09ec0?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e2ffa0 sp=0xc000e2ff78 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000e2ffe0 sp=0xc000e2ffa0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2ffe8 sp=0xc000e2ffe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 252 [chan receive, 808 minutes]:
runtime.gopark(0x20?, 0x23169e0?, 0x1?, 0x0?, 0xc000a56f68?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000a56ee8 sp=0xc000a56ec8 pc=0x43bd96
runtime.chanrecv(0xc001262900, 0xc000a56fc8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000a56f78 sp=0xc000a56ee8 pc=0x406f9d
runtime.chanrecv2(0x2e8ce00?, 0xc000e09f08?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000a56fa0 sp=0xc000a56f78 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66 fp=0xc000a56fe0 sp=0xc000a56fa0 pc=0x163ab86
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000a56fe8 sp=0xc000a56fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener
/__w/cortex/cortex/pkg/util/services/manager.go:243 +0x1d3
goroutine 253 [select, 810 minutes]:
runtime.gopark(0xc000582f40?, 0x2?, 0x0?, 0x0?, 0xc000582efc?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e0dd60 sp=0xc000e0dd40 pc=0x43bd96
runtime.selectgo(0xc000e0df40, 0xc000582ef8, 0x3?, 0x0, 0x0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000e0dea0 sp=0xc000e0dd60 pc=0x44c03e
github.com/weaveworks/common/signals.(*Handler).Loop(0xc001266360)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/signals/signals.go:48 +0x197 fp=0xc000e0dfa8 sp=0xc000e0dea0 pc=0x111fb37
github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run.func4()
/__w/cortex/cortex/pkg/cortex/cortex.go:448 +0x26 fp=0xc000e0dfe0 sp=0xc000e0dfa8 pc=0x21402e6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e0dfe8 sp=0xc000e0dfe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run
/__w/cortex/cortex/pkg/cortex/cortex.go:447 +0x85b
goroutine 254 [select, 808 minutes]:
runtime.gopark(0xc00100ddd8?, 0x2?, 0x0?, 0x0?, 0xc00100ddc4?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001203c40 sp=0xc001203c20 pc=0x43bd96
runtime.selectgo(0xc001203dd8, 0xc00100ddc0, 0x22f8d60?, 0x0, 0xc00100de98?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001203d80 sp=0xc001203c40 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0x1?, {0x2e95968, 0xc00109bf40}, 0x4, 0xc000e20c60) /__w/cortex/cortex/pkg/util/services/basic_service.go:294 +0x90 fp=0xc001203e58 sp=0xc001203d80 pc=0x1638530
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0x1?, {0x2e95968?, 0xc00109bf40?}) /__w/cortex/cortex/pkg/util/services/basic_service.go:290 +0x2c fp=0xc001203e90 sp=0xc001203e58 pc=0x163846c
github.com/cortexproject/cortex/pkg/compactor.(*Compactor).AwaitTerminated(0x1637732?, {0x2e95968?, 0xc00109bf40?}) <autogenerated>:1 +0x34 fp=0xc001203eb8 sp=0xc001203e90 pc=0x1df7b14
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run(0xc00109be00, {0x2e95968?, 0xc00109bf40?}) /__w/cortex/cortex/pkg/util/modules/module_service.go:76 +0x34 fp=0xc001203ee0 sp=0xc001203eb8 pc=0x20fec74
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run-fm({0x2e95968?, 0xc00109bf40?}) <autogenerated>:1 +0x39 fp=0xc001203f08 sp=0xc001203ee0 pc=0x2101699
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7b360)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc001203fc8 sp=0xc001203f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc001203fe0 sp=0xc001203fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001203fe8 sp=0xc001203fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 255 [select, 810 minutes]:
runtime.gopark(0xc0001c7e00?, 0x2?, 0x0?, 0x0?, 0xc0001c7dec?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001c7c68 sp=0xc0001c7c48 pc=0x43bd96
runtime.selectgo(0xc0001c7e00, 0xc0001c7de8, 0x406305?, 0x0, 0xc0009be0c0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0001c7da8 sp=0xc0001c7c68 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0x240e8e0?, {0x2e95968, 0xc00109bf90}, 0x4, 0xc000e202a0) /__w/cortex/cortex/pkg/util/services/basic_service.go:294 +0x90 fp=0xc0001c7e80 sp=0xc0001c7da8 pc=0x1638530
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0x1637732?, {0x2e95968?, 0xc00109bf90?}) /__w/cortex/cortex/pkg/util/services/basic_service.go:290 +0x2c fp=0xc0001c7eb8 sp=0xc0001c7e80 pc=0x163846c
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run(0xc00109ae10, {0x2e95968?, 0xc00109bf90?}) /__w/cortex/cortex/pkg/util/modules/module_service.go:76 +0x34 fp=0xc0001c7ee0 sp=0xc0001c7eb8 pc=0x20fec74
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run-fm({0x2e95968?, 0xc00109bf90?}) <autogenerated>:1 +0x39 fp=0xc0001c7f08 sp=0xc0001c7ee0 pc=0x2101699
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7a1e0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc0001c7fc8 sp=0xc0001c7f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc0001c7fe0 sp=0xc0001c7fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001c7fe8 sp=0xc0001c7fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 256 [select, 810 minutes]:
runtime.gopark(0xc001200dd8?, 0x2?, 0x0?, 0x0?, 0xc001200dc4?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001200c40 sp=0xc001200c20 pc=0x43bd96
runtime.selectgo(0xc001200dd8, 0xc001200dc0, 0x22f8d60?, 0x0, 0xc001200e98?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001200d80 sp=0xc001200c40 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0x1?, {0x2e95968, 0xc001268000}, 0x4, 0xc000e208a0) /__w/cortex/cortex/pkg/util/services/basic_service.go:294 +0x90 fp=0xc001200e58 sp=0xc001200d80 pc=0x1638530
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0x1?, {0x2e95968?, 0xc001268000?}) /__w/cortex/cortex/pkg/util/services/basic_service.go:290 +0x2c fp=0xc001200e90 sp=0xc001200e58 pc=0x163846c
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).AwaitTerminated(0x1637732?, {0x2e95968?, 0xc001268000?}) <autogenerated>:1 +0x34 fp=0xc001200eb8 sp=0xc001200e90 pc=0x18723d4
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run(0xc00109b220, {0x2e95968?, 0xc001268000?}) /__w/cortex/cortex/pkg/util/modules/module_service.go:76 +0x34 fp=0xc001200ee0 sp=0xc001200eb8 pc=0x20fec74
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run-fm({0x2e95968?, 0xc001268000?}) <autogenerated>:1 +0x39 fp=0xc001200f08 sp=0xc001200ee0 pc=0x2101699
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7ac80)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc001200fc8 sp=0xc001200f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc001200fe0 sp=0xc001200fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001200fe8 sp=0xc001200fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 257 [select, 810 minutes]:
runtime.gopark(0xc0005f5dd8?, 0x2?, 0x0?, 0x0?, 0xc0005f5dc4?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000a55c40 sp=0xc000a55c20 pc=0x43bd96
runtime.selectgo(0xc000a55dd8, 0xc0005f5dc0, 0x22f8d60?, 0x0, 0xc0005f5e98?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000a55d80 sp=0xc000a55c40 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0009be301?, {0x2e95968, 0xc001268050}, 0x4, 0xc000e20ae0) /__w/cortex/cortex/pkg/util/services/basic_service.go:294 +0x90 fp=0xc000a55e58 sp=0xc000a55d80 pc=0x1638530
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0x1?, {0x2e95968?, 0xc001268050?}) /__w/cortex/cortex/pkg/util/services/basic_service.go:290 +0x2c fp=0xc000a55e90 sp=0xc000a55e58 pc=0x163846c
github.com/cortexproject/cortex/pkg/util/runtimeconfig.(*Manager).AwaitTerminated(0x1637732?, {0x2e95968?, 0xc001268050?}) <autogenerated>:1 +0x34 fp=0xc000a55eb8 sp=0xc000a55e90 pc=0x212cfb4
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run(0xc00109b3b0, {0x2e95968?, 0xc001268050?}) /__w/cortex/cortex/pkg/util/modules/module_service.go:76 +0x34 fp=0xc000a55ee0 sp=0xc000a55eb8 pc=0x20fec74
github.com/cortexproject/cortex/pkg/util/modules.(*moduleService).run-fm({0x2e95968?, 0xc001268050?}) <autogenerated>:1 +0x39 fp=0xc000a55f08 sp=0xc000a55ee0 pc=0x2101699
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7b040)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc000a55fc8 sp=0xc000a55f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc000a55fe0 sp=0xc000a55fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000a55fe8 sp=0xc000a55fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 198 [select, 810 minutes]:
runtime.gopark(0xc000e29ed8?, 0x2?, 0x4a?, 0xdc?, 0xc000e29eb4?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e29d40 sp=0xc000e29d20 pc=0x43bd96
runtime.selectgo(0xc000e29ed8, 0xc000e29eb0, 0x0?, 0x0, 0x16377e6?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000e29e80 sp=0xc000e29d40 pc=0x44c03e
github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1({0x2e95968, 0xc00034f590}) /__w/cortex/cortex/pkg/cortex/server_service.go:28 +0xed fp=0xc000e29f08 sp=0xc000e29e80 pc=0x214a5ad
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7a140)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc000e29fc8 sp=0xc000e29f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc000e29fe0 sp=0xc000e29fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e29fe8 sp=0xc000e29fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 199 [chan receive, 810 minutes]:
runtime.gopark(0x23de7a0?, 0xff83e0?, 0x0?, 0xe5?, 0xc000585700?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000585680 sp=0xc000585660 pc=0x43bd96
runtime.chanrecv(0xc000f0b980, 0xc000585768, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000585710 sp=0xc000585680 pc=0x406f9d
runtime.chanrecv1(0x0?, 0x42a3560?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc000585738 sp=0xc000585710 pc=0x406a98
github.com/weaveworks/common/server.(*Server).Run(0xc000e64380)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:517 +0x1ba fp=0xc000585788 sp=0xc000585738 pc=0x1123dba
github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1.1()
/__w/cortex/cortex/pkg/cortex/server_service.go:25 +0x65 fp=0xc0005857e0 sp=0xc000585788 pc=0x214a685
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005857e8 sp=0xc0005857e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1
/__w/cortex/cortex/pkg/cortex/server_service.go:23 +0x8e
goroutine 200 [chan receive, 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000585ed0 sp=0xc000585eb0 pc=0x43bd96
runtime.chanrecv(0xc000e20120, 0x0, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000585f60 sp=0xc000585ed0 pc=0x406f9d
runtime.chanrecv1(0x0?, 0x0?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc000585f88 sp=0xc000585f60 pc=0x406a98
github.com/cortexproject/cortex/pkg/cortex.ignoreSignalHandler.Loop(0x0?)
/__w/cortex/cortex/pkg/cortex/server_service.go:65 +0x1b fp=0xc000585fa8 sp=0xc000585f88 pc=0x214a75b
github.com/weaveworks/common/server.(*Server).Run.func1()
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:477 +0x35 fp=0xc000585fe0 sp=0xc000585fa8 pc=0x1123fd5
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000585fe8 sp=0xc000585fe0 pc=0x46fd61
created by github.com/weaveworks/common/server.(*Server).Run
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:476 +0x8a
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001207bc0 sp=0xc001207ba0 pc=0x43bd96
runtime.netpollblock(0x7ffff7fc9268?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc001207bf8 sp=0xc001207bc0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd0ad2648, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc001207c18 sp=0xc001207bf8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc000c9c800?, 0x4?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001207c40 sp=0xc001207c18 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c9c800)
/usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd fp=0xc001207ce8 sp=0xc001207c40 pc=0x50369d
net.(*netFD).accept(0xc000c9c800)
/usr/local/go/src/net/fd_unix.go:172 +0x35 fp=0xc001207da0 sp=0xc001207ce8 pc=0x6f8e55
net.(*TCPListener).accept(0xc000e08438)
/usr/local/go/src/net/tcpsock_posix.go:148 +0x25 fp=0xc001207dc8 sp=0xc001207da0 pc=0x714665
net.(*TCPListener).Accept(0xc000e08438)
/usr/local/go/src/net/tcpsock.go:297 +0x3d fp=0xc001207df8 sp=0xc001207dc8 pc=0x71341d
github.com/weaveworks/common/middleware.(*countingListener).Accept(0xc000308ee0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/counting_listener.go:22 +0x2d fp=0xc001207e28 sp=0xc001207df8 pc=0x111558d
net/http.(*onceCloseListener).Accept(0xc0014ea090?)
<autogenerated>:1 +0x2a fp=0xc001207e40 sp=0xc001207e28 pc=0x835c6a
net/http.(*Server).Serve(0xc0002b0000, {0x2e91000, 0xc000308ee0}) /usr/local/go/src/net/http/server.go:3059 +0x385 fp=0xc001207f70 sp=0xc001207e40 pc=0x80fd65
github.com/weaveworks/common/server.(*Server).Run.func2()
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:487 +0x7a fp=0xc001207fe0 sp=0xc001207f70 pc=0x1123f3a
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001207fe8 sp=0xc001207fe0 pc=0x46fd61
created by github.com/weaveworks/common/server.(*Server).Run
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:484 +0xde
goroutine 202 [IO wait, 810 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0001c9be0 sp=0xc0001c9bc0 pc=0x43bd96
runtime.netpollblock(0xf749fe?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc0001c9c18 sp=0xc0001c9be0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd0ad2558, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc0001c9c38 sp=0xc0001c9c18 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc000c9c880?, 0x2?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0001c9c60 sp=0xc0001c9c38 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c9c880)
/usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd fp=0xc0001c9d08 sp=0xc0001c9c60 pc=0x50369d
net.(*netFD).accept(0xc000c9c880)
/usr/local/go/src/net/fd_unix.go:172 +0x35 fp=0xc0001c9dc0 sp=0xc0001c9d08 pc=0x6f8e55
net.(*TCPListener).accept(0xc000e08480)
/usr/local/go/src/net/tcpsock_posix.go:148 +0x25 fp=0xc0001c9de8 sp=0xc0001c9dc0 pc=0x714665
net.(*TCPListener).Accept(0xc000e08480)
/usr/local/go/src/net/tcpsock.go:297 +0x3d fp=0xc0001c9e18 sp=0xc0001c9de8 pc=0x71341d
github.com/weaveworks/common/middleware.(*countingListener).Accept(0xc000308f00)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/counting_listener.go:22 +0x2d fp=0xc0001c9e48 sp=0xc0001c9e18 pc=0x111558d
google.golang.org/grpc.(*Server).Serve(0xc000e3e3c0, {0x2e91000?, 0xc000308f00}) /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:823 +0x475 fp=0xc0001c9f90 sp=0xc0001c9e48 pc=0xff8f15
github.com/weaveworks/common/server.(*Server).Run.func3()
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:506 +0x3f fp=0xc0001c9fe0 sp=0xc0001c9f90 pc=0x1123e3f
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001c9fe8 sp=0xc0001c9fe0 pc=0x46fd61
created by github.com/weaveworks/common/server.(*Server).Run
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:505 +0x1a5
goroutine 258 [select, 810 minutes]:
runtime.gopark(0xc000e2a6b0?, 0x2?, 0x4a?, 0xdc?, 0xc000e2a69c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2a528 sp=0xc000e2a508 pc=0x43bd96
runtime.selectgo(0xc000e2a6b0, 0xc000e2a698, 0xc000e20840?, 0x0, 0x10?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000e2a668 sp=0xc000e2a528 pc=0x44c03e
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).running(0xc000c9d000, {0x2e95968?, 0xc001268190?}) /__w/cortex/cortex/pkg/ring/kv/memberlist/kv_init_service.go:78 +0x90 fp=0xc000e2a6e0 sp=0xc000e2a668 pc=0x185d730
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).running-fm({0x2e95968?, 0xc001268190?}) <autogenerated>:1 +0x39 fp=0xc000e2a708 sp=0xc000e2a6e0 pc=0x1873519
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7aa00)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc000e2a7c8 sp=0xc000e2a708 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc000e2a7e0 sp=0xc000e2a7c8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2a7e8 sp=0xc000e2a7e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 273 [select, 810 minutes, locked to thread]:
runtime.gopark(0xc00063ffa8?, 0x2?, 0x12?, 0xc1?, 0xc00063ffa4?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00063fe18 sp=0xc00063fdf8 pc=0x43bd96
runtime.selectgo(0xc00063ffa8, 0xc00063ffa0, 0x0?, 0x0, 0x0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00063ff58 sp=0xc00063fe18 pc=0x44c03e
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:1000 +0x1b0 fp=0xc00063ffe0 sp=0xc00063ff58 pc=0x466670
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00063ffe8 sp=0xc00063ffe0 pc=0x46fd61
created by runtime.ensureSigM
/usr/local/go/src/runtime/signal_unix.go:983 +0xbd
runtime.gopark(0xc000e11e88?, 0x2?, 0xb0?, 0xc2?, 0xc000e11e3c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e11ca8 sp=0xc000e11c88 pc=0x43bd96
runtime.selectgo(0xc000e11e88, 0xc000e11e38, 0x0?, 0x0, 0x2?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000e11de8 sp=0xc000e11ca8 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/runtimeconfig.(*Manager).loop(0xc000e7ad20, {0x2e95968, 0xc0005d6050}) /__w/cortex/cortex/pkg/util/runtimeconfig/manager.go:132 +0x215 fp=0xc000e11ee0 sp=0xc000e11de8 pc=0x212c0f5
github.com/cortexproject/cortex/pkg/util/runtimeconfig.(*Manager).loop-fm({0x2e95968?, 0xc0005d6050?}) <autogenerated>:1 +0x39 fp=0xc000e11f08 sp=0xc000e11ee0 pc=0x212d419
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7adc0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc000e11fc8 sp=0xc000e11f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc000e11fe0 sp=0xc000e11fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e11fe8 sp=0xc000e11fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 289 [semacquire]:
runtime.gopark(0xc0009c08d0?, 0x500?, 0x60?, 0x4c?, 0x2?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0031029b8 sp=0xc003102998 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0xc001e22228, 0x20?, 0x1, 0x0, 0x68?)
/usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc003102a20 sp=0xc0031029b8 pc=0x44d12f
sync.runtime_Semacquire(0xc003102b72?)
/usr/local/go/src/runtime/sema.go:62 +0x27 fp=0xc003102a58 sp=0xc003102a20 pc=0x46b867
sync.(*WaitGroup).Wait(0xc0015bc000?)
/usr/local/go/src/sync/waitgroup.go:116 +0x4b fp=0xc003102a80 sp=0xc003102a58 pc=0x47c20b
github.com/thanos-io/thanos/pkg/compact.(*BucketCompactor).Compact(0xc00104d780, {0x2e95968?, 0xc000cea0f0}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1337 +0x878 fp=0xc003102db8 sp=0xc003102a80 pc=0x1dd39d8
github.com/cortexproject/cortex/pkg/compactor.(*Compactor).compactUser(0xc00120b000, {0x2e95968, 0xc000cea0f0}, {0xc0010917a6, 0x2}) /__w/cortex/cortex/pkg/compactor/compactor.go:831 +0xe05 fp=0xc003103998 sp=0xc003102db8 pc=0x1dee545
github.com/cortexproject/cortex/pkg/compactor.(*Compactor).compactUserWithRetries(0xc00120b000, {0x2e95968, 0xc000cea0f0}, {0xc0010917a6, 0x2}) /__w/cortex/cortex/pkg/compactor/compactor.go:744 +0x125 fp=0xc003103a20 sp=0xc003103998 pc=0x1ded6c5
github.com/cortexproject/cortex/pkg/compactor.(*Compactor).compactUsers(0xc00120b000, {0x2e95968, 0xc000cea0f0}) /__w/cortex/cortex/pkg/compactor/compactor.go:688 +0x970 fp=0xc003103e18 sp=0xc003103a20 pc=0x1dec1f0
github.com/cortexproject/cortex/pkg/compactor.(*Compactor).running(0xc00120b000, {0x2e95968, 0xc000cea0f0}) /__w/cortex/cortex/pkg/compactor/compactor.go:598 +0xc8 fp=0xc003103ee0 sp=0xc003103e18 pc=0x1deb688
github.com/cortexproject/cortex/pkg/compactor.(*Compactor).running-fm({0x2e95968?, 0xc000cea0f0?}) <autogenerated>:1 +0x39 fp=0xc003103f08 sp=0xc003103ee0 pc=0x1df8039
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000e7b0e0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc003103fc8 sp=0xc003103f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc003103fe0 sp=0xc003103fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003103fe8 sp=0xc003103fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 345 [select, 809 minutes]:
runtime.gopark(0xc001709e88?, 0x2?, 0x4a?, 0xdc?, 0xc001709e3c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001204ca8 sp=0xc001204c88 pc=0x43bd96
runtime.selectgo(0xc001204e88, 0xc001709e38, 0xc000ceaeb0?, 0x0, 0x1?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001204de8 sp=0xc001204ca8 pc=0x44c03e
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KV).running(0xc000701400, {0x2e95968?, 0xc000ceaeb0}) /__w/cortex/cortex/pkg/ring/kv/memberlist/memberlist_client.go:491 +0x425 fp=0xc001204ee0 sp=0xc001204de8 pc=0x1861aa5
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KV).running-fm({0x2e95968?, 0xc000ceaeb0?}) <autogenerated>:1 +0x39 fp=0xc001204f08 sp=0xc001204ee0 pc=0x1873419
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000f1e3c0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc001204fc8 sp=0xc001204f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc001204fe0 sp=0xc001204fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001204fe8 sp=0xc001204fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 372 [chan receive, 810 minutes]:
runtime.gopark(0xc000f0a960?, 0xc000e297c8?, 0x80?, 0x6e?, 0xc000f2a5e8?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e296e8 sp=0xc000e296c8 pc=0x43bd96
runtime.chanrecv(0xc000a4ccc0, 0xc000e297c8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e29778 sp=0xc000e296e8 pc=0x406f9d
runtime.chanrecv2(0x2e98ee0?, 0xc000f28768?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e297a0 sp=0xc000e29778 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000e297e0 sp=0xc000e297a0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e297e8 sp=0xc000e297e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 373 [chan receive, 809 minutes]:
runtime.gopark(0xc0009be1e0?, 0xc000e2afc8?, 0xc8?, 0x61?, 0xc000f2a5e8?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2aee8 sp=0xc000e2aec8 pc=0x43bd96
runtime.chanrecv(0xc000a4cd20, 0xc000e2afc8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e2af78 sp=0xc000e2aee8 pc=0x406f9d
runtime.chanrecv2(0x2e98ee0?, 0xc000f28780?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e2afa0 sp=0xc000e2af78 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/basic_service.go:344 +0x66 fp=0xc000e2afe0 sp=0xc000e2afa0 pc=0x1638c66
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2afe8 sp=0xc000e2afe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:343 +0x1ca
goroutine 374 [chan receive, 809 minutes]:
runtime.gopark(0xc000f0a7e0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000e2b6e8 sp=0xc000e2b6c8 pc=0x43bd96
runtime.chanrecv(0xc000a4cd80, 0xc000e2b7c8, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000e2b778 sp=0xc000e2b6e8 pc=0x406f9d
runtime.chanrecv2(0x2e8ce00?, 0xc000f28798?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000e2b7a0 sp=0xc000e2b778 pc=0x406ad8
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1()
/__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66 fp=0xc000e2b7e0 sp=0xc000e2b7a0 pc=0x163ab86
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e2b7e8 sp=0xc000e2b7e0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener
/__w/cortex/cortex/pkg/util/services/manager.go:243 +0x1d3
runtime.gopark(0xc000bb7e68?, 0x6?, 0x80?, 0x83?, 0xc000bb7d34?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00254bb38 sp=0xc00254bb18 pc=0x43bd96
runtime.selectgo(0xc00254be68, 0xc000bb7d28, 0xc00007a028?, 0x0, 0x0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00254bc78 sp=0xc00254bb38 pc=0x44c03e
github.com/cortexproject/cortex/pkg/ring.(*Lifecycler).loop(0xc000701800, {0x2e95968, 0xc000ceb3b0}) /__w/cortex/cortex/pkg/ring/lifecycler.go:436 +0x2aa fp=0xc00254bee0 sp=0xc00254bc78 pc=0x188594a
github.com/cortexproject/cortex/pkg/ring.(*Lifecycler).loop-fm({0x2e95968?, 0xc000ceb3b0?}) <autogenerated>:1 +0x39 fp=0xc00254bf08 sp=0xc00254bee0 pc=0x18a3219
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000f1e5a0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc00254bfc8 sp=0xc00254bf08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc00254bfe0 sp=0xc00254bfc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00254bfe8 sp=0xc00254bfe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
runtime.gopark(0xc0005f1ba0?, 0x3?, 0xc4?, 0x20?, 0xc0005f1b1a?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0005f1978 sp=0xc0005f1958 pc=0x43bd96
runtime.selectgo(0xc0005f1ba0, 0xc0005f1b14, 0x14?, 0x0, 0xc000ce3e30?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0005f1ab8 sp=0xc0005f1978 pc=0x44c03e
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KV).WatchKey(0xc000701400, {0x2e95a10, 0xc000698030}, {0xc0010a8228, 0x14}, {0x2e9a9e0, 0xc000ce3e30}, 0xc000630a70) /__w/cortex/cortex/pkg/ring/kv/memberlist/memberlist_client.go:703 +0x2aa fp=0xc0005f1be8 sp=0xc0005f1ab8 pc=0x1863e8a
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*Client).WatchKey(0xc000ce3e48, {0x2e95a10, 0xc000698030}, {0xc0010a8228, 0x14}, 0x0?) /__w/cortex/cortex/pkg/ring/kv/memberlist/memberlist_client.go:94 +0x74 fp=0xc0005f1c40 sp=0xc0005f1be8 pc=0x185f894
github.com/cortexproject/cortex/pkg/ring/kv.(*prefixedKVClient).WatchKey(0xc001096ae0, {0x2e95a10, 0xc000698030}, {0x286be72?, 0xc0005f1cb8?}, 0x493997?) /__w/cortex/cortex/pkg/ring/kv/prefix.go:46 +0x73 fp=0xc0005f1c88 sp=0xc0005f1c40 pc=0x1878a33
github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey.func1({0x2e95a10?, 0xc000698030?}) /__w/cortex/cortex/pkg/ring/kv/metrics.go:99 +0x42 fp=0xc0005f1cc8 sp=0xc0005f1c88 pc=0x1875622
github.com/weaveworks/common/instrument.CollectedRequest({0x2e95968, 0xc000ceb400}, {0x2869bb1, 0x8}, {0x2e90fd0, 0xc000cc6130}, 0x286734b?, 0xc0005f1e00) /__w/cortex/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:167 +0x26a fp=0xc0005f1dc0 sp=0xc0005f1cc8 pc=0x11037aa
github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey({{0x2ea61d0?, 0xc001096ae0?}, 0xc000cc6130?}, {0x2e95968?, 0xc000ceb400?}, {0x286be72?, 0x24e0860?}, 0x1?) /__w/cortex/cortex/pkg/ring/kv/metrics.go:98 +0xb7 fp=0xc0005f1e48 sp=0xc0005f1dc0 pc=0x1875577
github.com/cortexproject/cortex/pkg/ring/kv.(*metrics).WatchKey(0xc000ce04b8?, {0x2e95968?, 0xc000ceb400?}, {0x286be72?, 0x1637732?}, 0xc000cdfa40?) <autogenerated>:1 +0x65 fp=0xc0005f1e98 sp=0xc0005f1e48 pc=0x1879605
github.com/cortexproject/cortex/pkg/ring.(*Ring).loop(0xc000ce02c0, {0x2e95968, 0xc000ceb400}) /__w/cortex/cortex/pkg/ring/ring.go:280 +0xbe fp=0xc0005f1ee0 sp=0xc0005f1e98 pc=0x189371e
github.com/cortexproject/cortex/pkg/ring.(*Ring).loop-fm({0x2e95968?, 0xc000ceb400?}) <autogenerated>:1 +0x39 fp=0xc0005f1f08 sp=0xc0005f1ee0 pc=0x18a3199
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000f1e6e0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc0005f1fc8 sp=0xc0005f1f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc0005f1fe0 sp=0xc0005f1fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005f1fe8 sp=0xc0005f1fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 208 [runnable]:
runtime.gcTrigger.test({0x0?, 0x0?, 0x0?}) /usr/local/go/src/runtime/mgc.go:547 +0xfb fp=0xc000e0be78 sp=0xc000e0be70 pc=0x41b49b
runtime.mallocgc(0x20, 0x255ff00, 0x1)
/usr/local/go/src/runtime/malloc.go:1171 +0x768 fp=0xc000e0bee0 sp=0xc000e0be78 pc=0x40dda8
runtime.newobject(0xc0009c02a0?)
/usr/local/go/src/runtime/malloc.go:1254 +0x27 fp=0xc000e0bf08 sp=0xc000e0bee0 pc=0x40e027
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*TCPTransport).tcpListen(0xc0007221e0, {0x2e92f20, 0xc0009c02a0}) /__w/cortex/cortex/pkg/ring/kv/memberlist/tcp_transport.go:225 +0x232 fp=0xc000e0bfb8 sp=0xc000e0bf08 pc=0x186d9d2
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.NewTCPTransport.func2()
/__w/cortex/cortex/pkg/ring/kv/memberlist/tcp_transport.go:178 +0x2e fp=0xc000e0bfe0 sp=0xc000e0bfb8 pc=0x186d72e
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000e0bfe8 sp=0xc000e0bfe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/ring/kv/memberlist.NewTCPTransport
/__w/cortex/cortex/pkg/ring/kv/memberlist/tcp_transport.go:178 +0x7d3
goroutine 385 [select, 2 minutes]:
runtime.gopark(0xc000582f98?, 0x2?, 0x0?, 0x0?, 0xc000582f7c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000582e00 sp=0xc000582de0 pc=0x43bd96
runtime.selectgo(0xc000582f98, 0xc000582f78, 0xc000e20e40?, 0x0, 0x2e8afe0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000582f40 sp=0xc000582e00 pc=0x44c03e
github.com/hashicorp/memberlist.(*Memberlist).streamListen(0xc00117c160)
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/net.go:211 +0x96 fp=0xc000582fc8 sp=0xc000582f40 pc=0x11c6cd6
github.com/hashicorp/memberlist.newMemberlist.func3()
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/memberlist.go:216 +0x26 fp=0xc000582fe0 sp=0xc000582fc8 pc=0x11c37a6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000582fe8 sp=0xc000582fe0 pc=0x46fd61
created by github.com/hashicorp/memberlist.newMemberlist
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/memberlist.go:216 +0x92c
runtime.gopark(0xc00149af98?, 0x2?, 0xa5?, 0xe9?, 0xc00149af8c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00149ae00 sp=0xc00149ade0 pc=0x43bd96
runtime.selectgo(0xc00149af98, 0xc00149af88, 0x49a7?, 0x0, 0x2e8a248?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00149af40 sp=0xc00149ae00 pc=0x44c03e
github.com/hashicorp/memberlist.(*Memberlist).packetListen(0xc00117c160)
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/net.go:314 +0xc5 fp=0xc00149afc8 sp=0xc00149af40 pc=0x11c7de5
github.com/hashicorp/memberlist.newMemberlist.func4()
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/memberlist.go:217 +0x26 fp=0xc00149afe0 sp=0xc00149afc8 pc=0x11c3746
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00149afe8 sp=0xc00149afe0 pc=0x46fd61
created by github.com/hashicorp/memberlist.newMemberlist
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/memberlist.go:217 +0x96f
runtime.gopark(0xc003f9ff18?, 0x2?, 0xe0?, 0x8a?, 0xc003f9ff14?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003777d98 sp=0xc003777d78 pc=0x43bd96
runtime.selectgo(0xc003777f18, 0xc003f9ff10, 0xa5f?, 0x0, 0x2?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc003777ed8 sp=0xc003777d98 pc=0x44c03e
github.com/hashicorp/memberlist.(*Memberlist).packetHandler(0xc00117c160)
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/net.go:437 +0x77 fp=0xc003777fc8 sp=0xc003777ed8 pc=0x11c8cb7
github.com/hashicorp/memberlist.newMemberlist.func5()
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/memberlist.go:218 +0x26 fp=0xc003777fe0 sp=0xc003777fc8 pc=0x11c36e6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003777fe8 sp=0xc003777fe0 pc=0x46fd61
created by github.com/hashicorp/memberlist.newMemberlist
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/memberlist.go:218 +0x9b2
runtime.gopark(0xc001757f58?, 0x2?, 0x14?, 0x0?, 0xc001757f4c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc002589dd8 sp=0xc002589db8 pc=0x43bd96
runtime.selectgo(0xc002589f58, 0xc001757f48, 0x0?, 0x0, 0x2?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc002589f18 sp=0xc002589dd8 pc=0x44c03e
github.com/hashicorp/memberlist.(*Memberlist).triggerFunc(0x0?, 0x12a05f200, 0xc0009be8a0, 0xc0011242a0, 0xc000346e60)
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:155 +0x11d fp=0xc002589fa8 sp=0xc002589f18 pc=0x11d3dfd
github.com/hashicorp/memberlist.(*Memberlist).schedule.func2()
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:121 +0x36 fp=0xc002589fe0 sp=0xc002589fa8 pc=0x11d3c56
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc002589fe8 sp=0xc002589fe0 pc=0x46fd61
created by github.com/hashicorp/memberlist.(*Memberlist).schedule
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:121 +0x218
goroutine 389 [select, 1 minutes]:
runtime.gopark(0xc000bc9f70?, 0x2?, 0x0?, 0x0?, 0xc000bc9f64?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001774de8 sp=0xc001774dc8 pc=0x43bd96
runtime.selectgo(0xc001774f70, 0xc000bc9f60, 0x0?, 0x0, 0x2?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001774f28 sp=0xc001774de8 pc=0x44c03e
github.com/hashicorp/memberlist.(*Memberlist).pushPullTrigger(0xc00117c160, 0xc0011242a0)
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:182 +0x134 fp=0xc001774fc0 sp=0xc001774f28 pc=0x11d3f94
github.com/hashicorp/memberlist.(*Memberlist).schedule.func3()
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:127 +0x2a fp=0xc001774fe0 sp=0xc001774fc0 pc=0x11d3bea
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001774fe8 sp=0xc001774fe0 pc=0x46fd61
created by github.com/hashicorp/memberlist.(*Memberlist).schedule
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:127 +0x345
goroutine 390 [runnable]:
runtime.gopark(0xc0021b5f58?, 0x2?, 0x27?, 0x0?, 0xc0021b5f4c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001a41dd8 sp=0xc001a41db8 pc=0x43bd96
runtime.selectgo(0xc001a41f58, 0xc0021b5f48, 0x0?, 0x0, 0x2?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001a41f18 sp=0xc001a41dd8 pc=0x44c03e
github.com/hashicorp/memberlist.(*Memberlist).triggerFunc(0x0?, 0xbebc200, 0xc0009be900, 0xc0011242a0, 0xc000346e70)
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:155 +0x11d fp=0xc001a41fa8 sp=0xc001a41f18 pc=0x11d3dfd
github.com/hashicorp/memberlist.(*Memberlist).schedule.func4()
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:133 +0x36 fp=0xc001a41fe0 sp=0xc001a41fa8 pc=0x11d3b96
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001a41fe8 sp=0xc001a41fe0 pc=0x46fd61
created by github.com/hashicorp/memberlist.(*Memberlist).schedule
/__w/cortex/cortex/vendor/github.com/hashicorp/memberlist/state.go:133 +0x498
goroutine 1796650 [chan send]:
runtime.gopark(0xc000c353b8?, 0x528835?, 0xc0?, 0x53?, 0xc000c35378?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000c352f8 sp=0xc000c352d8 pc=0x43bd96
runtime.chansend(0xc002bb80c0, 0xc000c353c0, 0x1, 0xc000c353c0?)
/usr/local/go/src/runtime/chan.go:259 +0x42e fp=0xc000c35380 sp=0xc000c352f8 pc=0x40618e
runtime.chansend1(0x268e3a0?, 0x40f201?)
/usr/local/go/src/runtime/chan.go:145 +0x1d fp=0xc000c353b0 sp=0xc000c35380 pc=0x405d3d
golang.org/x/sync/errgroup.(*Group).Go(0xc001eb57c0, 0xc001eb5980)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:68 +0x38 fp=0xc000c353d0 sp=0xc000c353b0 pc=0xd011b8
github.com/thanos-io/thanos/pkg/compact.(*Group).compact.func2(0x2403900?, 0xc001668fc0, {0xc002bf5ef0, 0x4f}, {0x2e95968?, 0xc0016f01e0}, 0xc002782900) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1027 +0xeb fp=0xc000c35400 sp=0xc000c353d0 pc=0x1dd1b6b
github.com/thanos-io/thanos/pkg/compact.(*Group).compact(0xc001668fc0, {0x2e95968, 0xc00384cbe0}, {0xc001a86140, 0x34}, {0x2e7c1a0, 0xc002007290}, {0x2e87ca0, 0xc000ccce40}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1061 +0x7f0 fp=0xc000c35c18 sp=0xc000c35400 pc=0x1dcf410
github.com/thanos-io/thanos/pkg/compact.(*Group).Compact.func2({0x2e95968?, 0xc00384cbe0?}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:775 +0x75 fp=0xc000c35cb0 sp=0xc000c35c18 pc=0x1dccef5
github.com/thanos-io/thanos/pkg/tracing.DoInSpanWithErr({0x2e95968?, 0xc00384cbe0?}, {0x287b822?, 0x9?}, 0xc0017c0e40, {0xc002bb8e40?, 0xc000c2c000?, 0x1?}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:82 +0xd0 fp=0xc000c35d50 sp=0xc000c35cb0 pc=0x1ccad10
github.com/thanos-io/thanos/pkg/compact.(*Group).Compact(0xc001668fc0, {0x2e95968, 0xc00384cbe0}, {0xc0015b2060, 0x1e}, {0x2e7c1a0, 0xc002007290}, {0x2e87ca0, 0xc000ccce40}) /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:774 +0x35c fp=0xc000c35ea8 sp=0xc000c35d50 pc=0x1dccc7c
github.com/thanos-io/thanos/pkg/compact.(*BucketCompactor).Compact.func2()
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1250 +0x165 fp=0xc000c35fe0 sp=0xc000c35ea8 pc=0x1dd4505
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000c35fe8 sp=0xc000c35fe0 pc=0x46fd61
created by github.com/thanos-io/thanos/pkg/compact.(*BucketCompactor).Compact
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go:1247 +0x90a
goroutine 1801339 [runnable]:
runtime.gopark(0xc001823788?, 0x2?, 0x4a?, 0xdc?, 0xc00182374c?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0018235a8 sp=0xc001823588 pc=0x43bd96
runtime.selectgo(0xc001823788, 0xc001823748, 0x2861a3f?, 0x0, 0xc0020c6270?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0018236e8 sp=0xc0018235a8 pc=0x44c03e
github.com/minio/minio-go/v7.(*Client).newRetryTimer.func2()
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:82 +0x1ca fp=0xc0018237e0 sp=0xc0018236e8 pc=0x158208a
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0018237e8 sp=0xc0018237e0 pc=0x46fd61
created by github.com/minio/minio-go/v7.(*Client).newRetryTimer
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:73 +0x16a
goroutine 1801852 [runnable]:
/usr/local/go/src/runtime/proc.go:321
runtime.gcAssistAlloc(0xc000cffd40)
/usr/local/go/src/runtime/mgcmark.go:495 +0x225 fp=0xc001e37378 sp=0xc001e37318 pc=0x41f7e5
runtime.deductAssistCredit(0xc?)
/usr/local/go/src/runtime/malloc.go:1217 +0x5a fp=0xc001e373a0 sp=0xc001e37378 pc=0x40defa
runtime.mallocgc(0x60, 0x25428a0, 0x1)
/usr/local/go/src/runtime/malloc.go:932 +0xd0 fp=0xc001e37408 sp=0xc001e373a0 pc=0x40d710
runtime.newobject(0x42c5f60?)
/usr/local/go/src/runtime/malloc.go:1254 +0x27 fp=0xc001e37430 sp=0xc001e37408 pc=0x40e027
regexp.newOnePassMachine(...)
/usr/local/go/src/regexp/exec.go:386
regexp.(*Regexp).doOnePass(0xc000891ea0, {0x0?, 0x0}, {0x0, 0x0, 0x0}, {0xc001091690, 0xb}, 0x0, 0x0, ...) /usr/local/go/src/regexp/exec.go:403 +0xbf fp=0xc001e374e8 sp=0xc001e37430 pc=0x56a0bf
regexp.(*Regexp).doExecute(0xc000c28060?, {0x0?, 0x0?}, {0x0?, 0xc0007e7600?, 0xc0007e7698?}, {0xc001091690?, 0x272eb00?}, 0xc003b0e001?, 0x0, ...) /usr/local/go/src/regexp/exec.go:532 +0x2a5 fp=0xc001e37598 sp=0xc001e374e8 pc=0x56abc5
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:533
github.com/minio/minio-go/v7/pkg/s3utils.checkBucketNameCommon({0xc001091690, 0xb}, 0x0) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go:377 +0x85 fp=0xc001e37610 sp=0xc001e37598 pc=0x14d2e25
github.com/minio/minio-go/v7/pkg/s3utils.CheckValidBucketName(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go:397
github.com/minio/minio-go/v7.(*Client).GetObject(0xc000a4e420, {0x2e95968, 0xc0016f05f0}, {0xc001091690, 0xb}, {0xc003b0e000, 0x2b}, {0x0, 0x0, {0x0, ...}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:34 +0xc5 fp=0xc001e37698 sp=0xc001e37610 pc=0x1543fe5
github.com/thanos-io/objstore/providers/s3.(*Bucket).getRange(0xc000ccc360, {0x2e95968, 0xc0016f05f0}, {0xc003b0e000, 0x2b}, 0x0, 0xffffffffffffffff) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:437 +0x1fe fp=0xc001e37810 sp=0xc001e37698 pc=0x163537e
github.com/thanos-io/objstore/providers/s3.(*Bucket).Get(0x40e027?, {0x2e95968?, 0xc0016f05f0?}, {0xc003b0e000?, 0xc0007e7898?}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:456 +0x32 fp=0xc001e37858 sp=0xc001e37810 pc=0x1635752
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get.func1()
/__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:142 +0x4d fp=0xc001e378a0 sp=0xc001e37858 pc=0x16502ed
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).retry(0xc000ce8140, {0x2e95968?, 0xc0016f05f0?}, 0xc001e379c0, {0xc003b0e060, 0x2f}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:114 +0x137 fp=0xc001e37960 sp=0xc001e378a0 pc=0x164fbf7
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get(0xc000ce8140, {0x2e95968, 0xc0016f05f0}, {0xc003b0e000, 0x2b}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:141 +0x145 fp=0xc001e37a10 sp=0xc001e37960 pc=0x1650225
github.com/thanos-io/objstore.(*metricBucket).Get(0xc000ce8180, {0x2e95968, 0xc0016f05f0}, {0xc003b0e000, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:506 +0xb1 fp=0xc001e37a78 sp=0xc001e37a10 pc=0x12d8251
github.com/thanos-io/objstore.TracingBucket.Get({{0x2ead140?, 0xc000ce8180?}}, {0x2e95968?, 0xc0016f05f0?}, {0xc003b0e000, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/tracing.go:36 +0x14d fp=0xc001e37b28 sp=0xc001e37a78 pc=0x12d9d0d
github.com/thanos-io/objstore.(*TracingBucket).Get(0x2?, {0x2e95968?, 0xc0016f05f0?}, {0xc003b0e000?, 0x28?}) <autogenerated>:1 +0x4f fp=0xc001e37b68 sp=0xc001e37b28 pc=0x12dbbcf
github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex.(*globalMarkersBucket).Get(0xc0007e7bf0?, {0x2e95968?, 0xc0016f05f0?}, {0xc003b0e000?, 0x4231c5?}) /__w/cortex/cortex/pkg/storage/tsdb/bucketindex/markers_bucket_client.go:85 +0x30 fp=0xc001e37ba0 sp=0xc001e37b68 pc=0x1dc4bf0
github.com/cortexproject/cortex/pkg/storage/bucket.(*PrefixedBucketClient).Get(0xc001808000, {0x2e95968, 0xc0016f05f0}, {0xc0032b4393?, 0x5d?}) /__w/cortex/cortex/pkg/storage/bucket/prefixed_bucket_client.go:58 +0x75 fp=0xc001e37be8 sp=0xc001e37ba0 pc=0x166f8d5
github.com/cortexproject/cortex/pkg/storage/bucket.(*SSEBucketClient).Get(0xc003b08000?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b4393?, 0x101e7dd0ad4a1?}) /__w/cortex/cortex/pkg/storage/bucket/sse_bucket_client.go:104 +0x31 fp=0xc001e37c20 sp=0xc001e37be8 pc=0x16702b1
github.com/thanos-io/objstore.DownloadFile({0x2e95968, 0xc0016f05f0}, {0x2e7a3a0?, 0xc0020ee050?}, {0x7fffd070d3a0, 0xc002006900}, {0xc0032b4393, 0x28}, {0xc003b08000, 0x5d}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:307 +0x1ef fp=0xc001e37da8 sp=0xc001e37c20 pc=0x12d544f
github.com/thanos-io/objstore.DownloadDir.func1.1()
/__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:365 +0x63d fp=0xc001e37f78 sp=0xc001e37da8 pc=0x12d6dbd
golang.org/x/sync/errgroup.(*Group).Go.func1()
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64 fp=0xc001e37fe0 sp=0xc001e37f78 pc=0xd012c4
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001e37fe8 sp=0xc001e37fe0 pc=0x46fd61
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 1802514 [chan receive]:
runtime.gopark(0x0?, 0xc001d15380?, 0x0?, 0x0?, 0xc002f4eb20?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc002f4eab8 sp=0xc002f4ea98 pc=0x43bd96
runtime.chanrecv(0xc002fdc840, 0xc002f4eeb0, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc002f4eb48 sp=0xc002f4eab8 pc=0x406f9d
runtime.chanrecv2(0x2e7f700?, 0xc0009903c0?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc002f4eb70 sp=0xc002f4eb48 pc=0x406ad8
github.com/minio/minio-go/v7.(*Client).GetObject.func1()
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:76 +0x1d6 fp=0xc002f4ffe0 sp=0xc002f4eb70 pc=0x15445d6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc002f4ffe8 sp=0xc002f4ffe0 pc=0x46fd61
created by github.com/minio/minio-go/v7.(*Client).GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x385
runtime.gopark(0x453?, 0xb?, 0x0?, 0x0?, 0xc?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0017038d0 sp=0xc0017038b0 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc001703908 sp=0xc0017038d0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd0ad20a8, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc001703928 sp=0xc001703908 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc000c9c200?, 0xc000ac8000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001703950 sp=0xc001703928 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000c9c200, {0xc000ac8000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc0017039e8 sp=0xc001703950 pc=0x4ff179
net.(*netFD).Read(0xc000c9c200, {0xc000ac8000?, 0x4fe286?, 0x7fffd0ad2190?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc001703a30 sp=0xc0017039e8 pc=0x6f6cc9
net.(*conn).Read(0xc0001ac260, {0xc000ac8000?, 0x0?, 0x0?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc001703a78 sp=0xc001703a30 pc=0x70ad65
net.(*TCPConn).Read(0xc143ae0516c1d336?, {0xc000ac8000?, 0x42ca460?, 0xc001133988?}) <autogenerated>:1 +0x29 fp=0xc001703aa8 sp=0xc001703a78 pc=0x71e1c9
github.com/weaveworks/common/middleware.(*countingListenerConn).Read(0xc001133980?, {0xc000ac8000?, 0xc00288db00?, 0x42ca460?}) <autogenerated>:1 +0x2f fp=0xc001703ad8 sp=0xc001703aa8 pc=0x111b20f
net/http.(*connReader).Read(0xc001133980, {0xc000ac8000, 0x1000, 0x1000}) /usr/local/go/src/net/http/server.go:782 +0x171 fp=0xc001703b28 sp=0xc001703ad8 pc=0x804eb1
bufio.(*Reader).fill(0xc000a4d320)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc001703b60 sp=0xc001703b28 pc=0x54d55f
bufio.(*Reader).Peek(0xc000a4d320, 0x4)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc001703b80 sp=0xc001703b60 pc=0x54d6bd
net/http.(*conn).serve(0xc000cd0510, {0x2e95a10, 0xc000a4b080}) /usr/local/go/src/net/http/server.go:2030 +0x77c fp=0xc001703fb8 sp=0xc001703b80 pc=0x80ae7c
net/http.(*Server).Serve.func3()
/usr/local/go/src/net/http/server.go:3089 +0x2e fp=0xc001703fe0 sp=0xc001703fb8 pc=0x81014e
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001703fe8 sp=0xc001703fe0 pc=0x46fd61
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3089 +0x5ed
goroutine 1799597 [select]:
runtime.gopark(0xc00258cf68?, 0x4?, 0x3?, 0x0?, 0xc00258cdb0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00258cc00 sp=0xc00258cbe0 pc=0x43bd96
runtime.selectgo(0xc00258cf68, 0xc00258cda8, 0xc000990380?, 0x0, 0xc00258cf01?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00258cd40 sp=0xc00258cc00 pc=0x44c03e
net/http.(*persistConn).readLoop(0xc002829d40)
/usr/local/go/src/net/http/transport.go:2227 +0xd85 fp=0xc00258cfc8 sp=0xc00258cd40 pc=0x826a65
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc00258cfe0 sp=0xc00258cfc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00258cfe8 sp=0xc00258cfe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1800955 [chan receive]:
runtime.gopark(0x0?, 0x2?, 0x30?, 0x0?, 0x18?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0029e0e80 sp=0xc0029e0e60 pc=0x43bd96
runtime.chanrecv(0xc002f549c0, 0xc0029e1178, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc0029e0f10 sp=0xc0029e0e80 pc=0x406f9d
runtime.chanrecv1(0xc0029e0f70?, 0xc0029e0f68?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc0029e0f38 sp=0xc0029e0f10 pc=0x406a98
github.com/minio/minio-go/v7.(*Object).doGetRequest(_, {{0xc003900000, 0x8000, 0x8000}, 0x8000, 0x0, 0x1, 0x0, 0x1, 0x0, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:310 +0x113 fp=0xc0029e1348 sp=0xc0029e0f38 pc=0x15458b3
github.com/minio/minio-go/v7.(*Object).Read(0xc0025e8600, {0xc003900000?, 0x8000?, 0x8000?}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:384 +0x245 fp=0xc0029e19e0 sp=0xc0029e1348 pc=0x1545cc5
github.com/thanos-io/objstore.(*timingReadCloser).Read(0xc0008ce0e0, {0xc003900000?, 0x512daf?, 0xc000ca0008?}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:650 +0x32 fp=0xc0029e1a30 sp=0xc0029e19e0 pc=0x12d96f2
github.com/thanos-io/objstore.(*tracingReadCloser).Read(0xc00189e180, {0xc003900000?, 0xc003390000?, 0xc0029e1b30?}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/tracing.go:138 +0x32 fp=0xc0029e1a78 sp=0xc0029e1a30 pc=0x12db1d2
io.copyBuffer({0x2e83200, 0xc003390000}, {0x7fffd0421a00, 0xc00189e180}, {0x0, 0x0, 0x0}) /usr/local/go/src/io/io.go:427 +0x1b2 fp=0xc0029e1af8 sp=0xc0029e1a78 pc=0x4a1fd2
/usr/local/go/src/io/io.go:386
os.genericReadFrom(0xc0029e1b90?, {0x7fffd0421a00, 0xc00189e180}) /usr/local/go/src/os/file.go:161 +0x67 fp=0xc0029e1b50 sp=0xc0029e1af8 pc=0x50b527
os.(*File).ReadFrom(0xc000ca0008, {0x7fffd0421a00, 0xc00189e180}) /usr/local/go/src/os/file.go:155 +0x1b0 fp=0xc0029e1ba0 sp=0xc0029e1b50 pc=0x50b470
io.copyBuffer({0x2e7faa0, 0xc000ca0008}, {0x7fffd0421a00, 0xc00189e180}, {0x0, 0x0, 0x0}) /usr/local/go/src/io/io.go:413 +0x14b fp=0xc0029e1c20 sp=0xc0029e1ba0 pc=0x4a1f6b
/usr/local/go/src/io/io.go:386
github.com/thanos-io/objstore.DownloadFile({0x2e95968, 0xc001830320}, {0x2e7a3a0?, 0xc0020ee050?}, {0x7fffd070d3a0, 0xc002006900}, {0xc000f07113, 0x28}, {0xc002f548a0, 0x5d}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:326 +0x466 fp=0xc0029e1da8 sp=0xc0029e1c20 pc=0x12d56c6
github.com/thanos-io/objstore.DownloadDir.func1.1()
/__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:365 +0x63d fp=0xc0029e1f78 sp=0xc0029e1da8 pc=0x12d6dbd
golang.org/x/sync/errgroup.(*Group).Go.func1()
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64 fp=0xc0029e1fe0 sp=0xc0029e1f78 pc=0xd012c4
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0029e1fe8 sp=0xc0029e1fe0 pc=0x46fd61
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 1785581 [IO wait]:
runtime.gopark(0xc001775b18?, 0xb?, 0x0?, 0x0?, 0x1a?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001775ab0 sp=0xc001775a90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc001775ae8 sp=0xc001775ab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd0ad1a18, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc001775b08 sp=0xc001775ae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc000f12480?, 0xc001051000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001775b30 sp=0xc001775b08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000f12480, {0xc001051000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc001775bc8 sp=0xc001775b30 pc=0x4ff179
net.(*netFD).Read(0xc000f12480, {0xc001051000?, 0x43be47?, 0xc001775c30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc001775c10 sp=0xc001775bc8 pc=0x6f6cc9
net.(*conn).Read(0xc0000121f8, {0xc001051000?, 0x0?, 0xc00264c340?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc001775c58 sp=0xc001775c10 pc=0x70ad65
net.(*TCPConn).Read(0xc001bae0a0?, {0xc001051000?, 0xc001bb80d0?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc001775c88 sp=0xc001775c58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc001086120, {0xc001051000?, 0xc001b00360?, 0xc001775d30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc001775ce8 sp=0xc001775c88 pc=0x82506e
bufio.(*Reader).fill(0xc0007f4de0)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc001775d20 sp=0xc001775ce8 pc=0x54d55f
bufio.(*Reader).Peek(0xc0007f4de0, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc001775d40 sp=0xc001775d20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc001086120)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc001775fc8 sp=0xc001775d40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc001775fe0 sp=0xc001775fc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001775fe8 sp=0xc001775fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 3304 [select, 108 minutes]:
runtime.gopark(0xc00100ded0?, 0x2?, 0x50?, 0xdd?, 0xc00100de94?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001763d18 sp=0xc001763cf8 pc=0x43bd96
runtime.selectgo(0xc001763ed0, 0xc00100de90, 0xc001387eb0?, 0x0, 0xc000cec420?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001763e58 sp=0xc001763d18 pc=0x44c03e
github.com/cortexproject/cortex/pkg/util/services.NewTimerService.func1({0x2e95968, 0xc0005d6d70}) /__w/cortex/cortex/pkg/util/services/services.go:31 +0xf7 fp=0xc001763f08 sp=0xc001763e58 pc=0x163b677
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000f1e000)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1e8 fp=0xc001763fc8 sp=0xc001763f08 pc=0x1637a08
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1.2()
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x26 fp=0xc001763fe0 sp=0xc001763fc8 pc=0x1637626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001763fe8 sp=0xc001763fe0 pc=0x46fd61
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x10a
goroutine 1800956 [runnable]:
runtime.gopark(0xc001c028a0?, 0xc0018a3e60?, 0x0?, 0x0?, 0xc003488b20?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003488ab8 sp=0xc003488a98 pc=0x43bd96
runtime.chanrecv(0xc002f54960, 0xc003488eb0, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc003488b48 sp=0xc003488ab8 pc=0x406f9d
runtime.chanrecv2(0x2e7f700?, 0xc00189e140?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc003488b70 sp=0xc003488b48 pc=0x406ad8
github.com/minio/minio-go/v7.(*Client).GetObject.func1()
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:76 +0x1d6 fp=0xc003489fe0 sp=0xc003488b70 pc=0x15445d6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003489fe8 sp=0xc003489fe0 pc=0x46fd61
created by github.com/minio/minio-go/v7.(*Client).GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x385
goroutine 1802359 [semacquire]:
runtime.gopark(0xc003797b00?, 0x4231c5?, 0x60?, 0xd9?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003797ac0 sp=0xc003797aa0 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0x42caed8, 0x87?, 0x0, 0x0, 0x88?)
/usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc003797b28 sp=0xc003797ac0 pc=0x44d12f
/usr/local/go/src/runtime/sema.go:111
runtime.gcStart({0x7ffff7fbf688?, 0x286dc02?, 0x1c20180?}) /usr/local/go/src/runtime/mgc.go:607 +0xf5 fp=0xc003797bb0 sp=0xc003797b28 pc=0x41b5b5
runtime.mallocgc(0x8, 0x27f4fc0, 0x1)
/usr/local/go/src/runtime/malloc.go:1172 +0x777 fp=0xc003797c18 sp=0xc003797bb0 pc=0x40ddb7
runtime.newobject(0xc001c200f0?)
/usr/local/go/src/runtime/malloc.go:1254 +0x27 fp=0xc003797c40 sp=0xc003797c18 pc=0x40e027
net/http.(*Client).makeHeadersCopier(0xc000cc43f0, 0xc003424100)
/usr/local/go/src/net/http/client.go:755 +0xd8 fp=0xc003797cd0 sp=0xc003797c40 pc=0x7c1918
net/http.(*Client).do(0xc000cc43f0, 0xc003424100)
/usr/local/go/src/net/http/client.go:603 +0x276 fp=0xc003797ed8 sp=0xc003797cd0 pc=0x7c07f6
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:582
github.com/minio/minio-go/v7.(*Client).do(0xc000a4e420, 0xc001091690?)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:504 +0xb3 fp=0xc003798040 sp=0xc003797ed8 pc=0x15778d3
github.com/minio/minio-go/v7.(*Client).executeMethod(0xc000a4e420, {0x2e95968, 0xc0030b2000}, {0x28611f9, 0x3}, {0x0, {0xc001091690, 0xb}, {0xc0022ea000, 0x2b}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:620 +0x972 fp=0xc003798730 sp=0xc003798040 pc=0x1578672
github.com/minio/minio-go/v7.(*Client).getObject(_, {_, _}, {_, _}, {_, _}, {0x0, 0x0, {0x0, ...}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:656 +0x1fc fp=0xc003798b70 sp=0xc003798730 pc=0x1547f3c
github.com/minio/minio-go/v7.(*Client).GetObject.func1()
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:91 +0x330 fp=0xc003799fe0 sp=0xc003798b70 pc=0x1544730
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003799fe8 sp=0xc003799fe0 pc=0x46fd61
created by github.com/minio/minio-go/v7.(*Client).GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x385
goroutine 1801352 [chan receive]:
runtime.gopark(0x1?, 0x2?, 0x0?, 0x0?, 0x49f826?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00127ab38 sp=0xc00127ab18 pc=0x43bd96
runtime.chanrecv(0xc002d22180, 0xc00127ae30, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc00127abc8 sp=0xc00127ab38 pc=0x406f9d
runtime.chanrecv1(0xc00127ac28?, 0xc00127ac20?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc00127abf0 sp=0xc00127abc8 pc=0x406a98
github.com/minio/minio-go/v7.(*Object).doGetRequest(_, {{0x0, 0x0, 0x0}, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:310 +0x113 fp=0xc00127b000 sp=0xc00127abf0 pc=0x15458b3
github.com/minio/minio-go/v7.(*Object).Read(0xc0001fe200, {0x0?, 0x0?, 0x0?}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:384 +0x245 fp=0xc00127b698 sp=0xc00127b000 pc=0x1545cc5
github.com/thanos-io/objstore/providers/s3.(*Bucket).getRange(0xc000ccc360, {0x2e95968, 0xc0016f05f0}, {0xc0032b47e0, 0x2b}, 0x0, 0xffffffffffffffff) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:444 +0x21d fp=0xc00127b810 sp=0xc00127b698 pc=0x163539d
github.com/thanos-io/objstore/providers/s3.(*Bucket).Get(0x40e027?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b47e0?, 0xc001a40898?}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:456 +0x32 fp=0xc00127b858 sp=0xc00127b810 pc=0x1635752
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get.func1()
/__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:142 +0x4d fp=0xc00127b8a0 sp=0xc00127b858 pc=0x16502ed
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).retry(0xc000ce8140, {0x2e95968?, 0xc0016f05f0?}, 0xc00127b9c0, {0xc0032b4840, 0x2f}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:114 +0x137 fp=0xc00127b960 sp=0xc00127b8a0 pc=0x164fbf7
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get(0xc000ce8140, {0x2e95968, 0xc0016f05f0}, {0xc0032b47e0, 0x2b}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:141 +0x145 fp=0xc00127ba10 sp=0xc00127b960 pc=0x1650225
github.com/thanos-io/objstore.(*metricBucket).Get(0xc000ce8180, {0x2e95968, 0xc0016f05f0}, {0xc0032b47e0, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:506 +0xb1 fp=0xc00127ba78 sp=0xc00127ba10 pc=0x12d8251
github.com/thanos-io/objstore.TracingBucket.Get({{0x2ead140?, 0xc000ce8180?}}, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b47e0, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/tracing.go:36 +0x14d fp=0xc00127bb28 sp=0xc00127ba78 pc=0x12d9d0d
github.com/thanos-io/objstore.(*TracingBucket).Get(0x2?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b47e0?, 0x28?}) <autogenerated>:1 +0x4f fp=0xc00127bb68 sp=0xc00127bb28 pc=0x12dbbcf
github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex.(*globalMarkersBucket).Get(0xc001a40bf0?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b47e0?, 0x194?}) /__w/cortex/cortex/pkg/storage/tsdb/bucketindex/markers_bucket_client.go:85 +0x30 fp=0xc00127bba0 sp=0xc00127bb68 pc=0x1dc4bf0
github.com/cortexproject/cortex/pkg/storage/bucket.(*PrefixedBucketClient).Get(0xc001808000, {0x2e95968, 0xc0016f05f0}, {0xc0032b4273?, 0x5d?}) /__w/cortex/cortex/pkg/storage/bucket/prefixed_bucket_client.go:58 +0x75 fp=0xc00127bbe8 sp=0xc00127bba0 pc=0x166f8d5
github.com/cortexproject/cortex/pkg/storage/bucket.(*SSEBucketClient).Get(0xc002d22060?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b4273?, 0x100c00628b4a8?}) /__w/cortex/cortex/pkg/storage/bucket/sse_bucket_client.go:104 +0x31 fp=0xc00127bc20 sp=0xc00127bbe8 pc=0x16702b1
github.com/thanos-io/objstore.DownloadFile({0x2e95968, 0xc0016f05f0}, {0x2e7a3a0?, 0xc0020ee050?}, {0x7fffd070d3a0, 0xc002006900}, {0xc0032b4273, 0x28}, {0xc002d22060, 0x5d}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:307 +0x1ef fp=0xc00127bda8 sp=0xc00127bc20 pc=0x12d544f
github.com/thanos-io/objstore.DownloadDir.func1.1()
/__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:365 +0x63d fp=0xc00127bf78 sp=0xc00127bda8 pc=0x12d6dbd
golang.org/x/sync/errgroup.(*Group).Go.func1()
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64 fp=0xc00127bfe0 sp=0xc00127bf78 pc=0xd012c4
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00127bfe8 sp=0xc00127bfe0 pc=0x46fd61
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 1785245 [select]:
runtime.gopark(0xc001967f90?, 0x2?, 0xf8?, 0x7d?, 0xc001967f34?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001967db0 sp=0xc001967d90 pc=0x43bd96
runtime.selectgo(0xc001967f90, 0xc001967f30, 0xc001b14080?, 0x0, 0xc002df8090?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001967ef0 sp=0xc001967db0 pc=0x44c03e
net/http.(*persistConn).writeLoop(0xc0005ae240)
/usr/local/go/src/net/http/transport.go:2410 +0xf2 fp=0xc001967fc8 sp=0xc001967ef0 pc=0x827b52
net/http.(*Transport).dialConn.func6()
/usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc001967fe0 sp=0xc001967fc8 pc=0x8245c6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001967fe8 sp=0xc001967fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1766 +0x173d
goroutine 1801853 [semacquire]:
runtime.gopark(0xc0016734d0?, 0x4231c5?, 0x60?, 0xd9?, 0x2e9a920?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001673490 sp=0xc001673470 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0x42caed8, 0x87?, 0x0, 0x0, 0x58?)
/usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc0016734f8 sp=0xc001673490 pc=0x44d12f
/usr/local/go/src/runtime/sema.go:111
runtime.gcStart({0xc003d9a000?, 0x1a0?, 0x1a0?}) /usr/local/go/src/runtime/mgc.go:607 +0xf5 fp=0xc001673580 sp=0xc0016734f8 pc=0x41b5b5
runtime.mallocgc(0x1a0, 0x28176c0, 0x1)
/usr/local/go/src/runtime/malloc.go:1172 +0x777 fp=0xc0016735e8 sp=0xc001673580 pc=0x40ddb7
runtime.newobject(0x2e95968?)
/usr/local/go/src/runtime/malloc.go:1254 +0x27 fp=0xc001673610 sp=0xc0016735e8 pc=0x40e027
github.com/minio/minio-go/v7.(*Client).GetObject(0xc000a4e420, {0x2e95968, 0xc0016f05f0}, {0xc001091690, 0xb}, {0xc003d94000, 0x2b}, {0x0, 0x0, {0x0, ...}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:52 +0x19a fp=0xc001673698 sp=0xc001673610 pc=0x15440ba
github.com/thanos-io/objstore/providers/s3.(*Bucket).getRange(0xc000ccc360, {0x2e95968, 0xc0016f05f0}, {0xc003d94000, 0x2b}, 0x0, 0xffffffffffffffff) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:437 +0x1fe fp=0xc001673810 sp=0xc001673698 pc=0x163537e
github.com/thanos-io/objstore/providers/s3.(*Bucket).Get(0x40e027?, {0x2e95968?, 0xc0016f05f0?}, {0xc003d94000?, 0xc00258b898?}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:456 +0x32 fp=0xc001673858 sp=0xc001673810 pc=0x1635752
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get.func1()
/__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:142 +0x4d fp=0xc0016738a0 sp=0xc001673858 pc=0x16502ed
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).retry(0xc000ce8140, {0x2e95968?, 0xc0016f05f0?}, 0xc0016739c0, {0xc003d94060, 0x2f}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:114 +0x137 fp=0xc001673960 sp=0xc0016738a0 pc=0x164fbf7
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get(0xc000ce8140, {0x2e95968, 0xc0016f05f0}, {0xc003d94000, 0x2b}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:141 +0x145 fp=0xc001673a10 sp=0xc001673960 pc=0x1650225
github.com/thanos-io/objstore.(*metricBucket).Get(0xc000ce8180, {0x2e95968, 0xc0016f05f0}, {0xc003d94000, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:506 +0xb1 fp=0xc001673a78 sp=0xc001673a10 pc=0x12d8251
github.com/thanos-io/objstore.TracingBucket.Get({{0x2ead140?, 0xc000ce8180?}}, {0x2e95968?, 0xc0016f05f0?}, {0xc003d94000, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/tracing.go:36 +0x14d fp=0xc001673b28 sp=0xc001673a78 pc=0x12d9d0d
github.com/thanos-io/objstore.(*TracingBucket).Get(0x2?, {0x2e95968?, 0xc0016f05f0?}, {0xc003d94000?, 0x28?}) <autogenerated>:1 +0x4f fp=0xc001673b68 sp=0xc001673b28 pc=0x12dbbcf
github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex.(*globalMarkersBucket).Get(0xc00258bbf0?, {0x2e95968?, 0xc0016f05f0?}, {0xc003d94000?, 0x4231c5?}) /__w/cortex/cortex/pkg/storage/tsdb/bucketindex/markers_bucket_client.go:85 +0x30 fp=0xc001673ba0 sp=0xc001673b68 pc=0x1dc4bf0
github.com/cortexproject/cortex/pkg/storage/bucket.(*PrefixedBucketClient).Get(0xc001808000, {0x2e95968, 0xc0016f05f0}, {0xc0032b4453?, 0x5d?}) /__w/cortex/cortex/pkg/storage/bucket/prefixed_bucket_client.go:58 +0x75 fp=0xc001673be8 sp=0xc001673ba0 pc=0x166f8d5
github.com/cortexproject/cortex/pkg/storage/bucket.(*SSEBucketClient).Get(0xc003d92000?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b4453?, 0x101c0002ac010?}) /__w/cortex/cortex/pkg/storage/bucket/sse_bucket_client.go:104 +0x31 fp=0xc001673c20 sp=0xc001673be8 pc=0x16702b1
github.com/thanos-io/objstore.DownloadFile({0x2e95968, 0xc0016f05f0}, {0x2e7a3a0?, 0xc0020ee050?}, {0x7fffd070d3a0, 0xc002006900}, {0xc0032b4453, 0x28}, {0xc003d92000, 0x5d}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:307 +0x1ef fp=0xc001673da8 sp=0xc001673c20 pc=0x12d544f
github.com/thanos-io/objstore.DownloadDir.func1.1()
/__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:365 +0x63d fp=0xc001673f78 sp=0xc001673da8 pc=0x12d6dbd
golang.org/x/sync/errgroup.(*Group).Go.func1()
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64 fp=0xc001673fe0 sp=0xc001673f78 pc=0xd012c4
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001673fe8 sp=0xc001673fe0 pc=0x46fd61
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 1785735 [select]:
runtime.gopark(0xc001960f90?, 0x2?, 0xf8?, 0xd?, 0xc001960f34?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001960db0 sp=0xc001960d90 pc=0x43bd96
runtime.selectgo(0xc001960f90, 0xc001960f30, 0xc000fb21c0?, 0x0, 0xc000aa35c0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001960ef0 sp=0xc001960db0 pc=0x44c03e
net/http.(*persistConn).writeLoop(0xc000c74000)
/usr/local/go/src/net/http/transport.go:2410 +0xf2 fp=0xc001960fc8 sp=0xc001960ef0 pc=0x827b52
net/http.(*Transport).dialConn.func6()
/usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc001960fe0 sp=0xc001960fc8 pc=0x8245c6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001960fe8 sp=0xc001960fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1766 +0x173d
goroutine 1802531 [semacquire]:
runtime.gopark(0xc003812f60?, 0x4231c5?, 0x60?, 0xd9?, 0x0?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003812f20 sp=0xc003812f00 pc=0x43bd96
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0x42caed8, 0x87?, 0x0, 0x0, 0xe8?)
/usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc003812f88 sp=0xc003812f20 pc=0x44d12f
/usr/local/go/src/runtime/sema.go:111
runtime.gcStart({0x7ffff7fbfb38?, 0x7?, 0x0?}) /usr/local/go/src/runtime/mgc.go:607 +0xf5 fp=0xc003813010 sp=0xc003812f88 pc=0x41b5b5
runtime.mallocgc(0x16, 0x0, 0x0)
/usr/local/go/src/runtime/malloc.go:1172 +0x777 fp=0xc003813078 sp=0xc003813010 pc=0x40ddb7
/usr/local/go/src/runtime/string.go:267
runtime.rawstringtmp(0x0?, 0x16)
/usr/local/go/src/runtime/string.go:131 +0x46 fp=0xc0038130a0 sp=0xc003813078 pc=0x455406
runtime.concatstrings(0xe?, {0xc003813130?, 0x4, 0x703088?}) /usr/local/go/src/runtime/string.go:51 +0xe9 fp=0xc003813110 sp=0xc0038130a0 pc=0x454da9
runtime.concatstring4(0xc000cda067?, {0xc000cda060?, 0x0?}, {0x2860fb9?, 0x0?}, {0xc000cda067?, 0xe?}, {0x2e67240?, 0x0?}) /usr/local/go/src/runtime/string.go:68 +0x87 fp=0xc003813180 sp=0xc003813110 pc=0x455107
github.com/minio/minio-go/v7.(*Client).makeTargetURL(0xc000a4e420, {0xc001091690, 0xb}, {0xc002ae8000, 0x2b}, {0x286cc4c, 0x9}, 0x0, 0xc00162c000) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:918 +0x91c fp=0xc003813668 sp=0xc003813180 pc=0x157af7c
github.com/minio/minio-go/v7.(*Client).newRequest(0xc000a4e420, {0x2e95968, 0xc00013c190}, {0x28611f9?, 0x0?}, {0x0, {0xc001091690, 0xb}, {0xc002ae8000, 0x2b}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:742 +0x2fa fp=0xc003814040 sp=0xc003813668 pc=0x157943a
github.com/minio/minio-go/v7.(*Client).executeMethod(0xc000a4e420, {0x2e95968, 0xc00013c190}, {0x28611f9, 0x3}, {0x0, {0xc001091690, 0xb}, {0xc002ae8000, 0x2b}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:609 +0x7ea fp=0xc003814730 sp=0xc003814040 pc=0x15784ea
github.com/minio/minio-go/v7.(*Client).getObject(_, {_, _}, {_, _}, {_, _}, {0x0, 0x0, {0x0, ...}, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:656 +0x1fc fp=0xc003814b70 sp=0xc003814730 pc=0x1547f3c
github.com/minio/minio-go/v7.(*Client).GetObject.func1()
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:91 +0x330 fp=0xc003815fe0 sp=0xc003814b70 pc=0x1544730
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003815fe8 sp=0xc003815fe0 pc=0x46fd61
created by github.com/minio/minio-go/v7.(*Client).GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x385
goroutine 41878 [IO wait]:
runtime.gopark(0x4b8?, 0xb?, 0x0?, 0x0?, 0xb?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00254f8d0 sp=0xc00254f8b0 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc00254f908 sp=0xc00254f8d0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd0ad1ec8, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc00254f928 sp=0xc00254f908 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc0005f7100?, 0xc001444000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00254f950 sp=0xc00254f928 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005f7100, {0xc001444000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc00254f9e8 sp=0xc00254f950 pc=0x4ff179
net.(*netFD).Read(0xc0005f7100, {0xc001444000?, 0x4fe286?, 0x7fffd0ad1fb0?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc00254fa30 sp=0xc00254f9e8 pc=0x6f6cc9
net.(*conn).Read(0xc0000127d0, {0xc001444000?, 0x0?, 0x0?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc00254fa78 sp=0xc00254fa30 pc=0x70ad65
net.(*TCPConn).Read(0xc143adf7d99bc165?, {0xc001444000?, 0x42ca460?, 0xc000e1c908?}) <autogenerated>:1 +0x29 fp=0xc00254faa8 sp=0xc00254fa78 pc=0x71e1c9
github.com/weaveworks/common/middleware.(*countingListenerConn).Read(0xc000e1c900?, {0xc001444000?, 0xc001a89b00?, 0x42ca460?}) <autogenerated>:1 +0x2f fp=0xc00254fad8 sp=0xc00254faa8 pc=0x111b20f
net/http.(*connReader).Read(0xc000e1c900, {0xc001444000, 0x1000, 0x1000}) /usr/local/go/src/net/http/server.go:782 +0x171 fp=0xc00254fb28 sp=0xc00254fad8 pc=0x804eb1
bufio.(*Reader).fill(0xc0014c8960)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc00254fb60 sp=0xc00254fb28 pc=0x54d55f
bufio.(*Reader).Peek(0xc0014c8960, 0x4)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc00254fb80 sp=0xc00254fb60 pc=0x54d6bd
net/http.(*conn).serve(0xc000378990, {0x2e95a10, 0xc000a4b080}) /usr/local/go/src/net/http/server.go:2030 +0x77c fp=0xc00254ffb8 sp=0xc00254fb80 pc=0x80ae7c
net/http.(*Server).Serve.func3()
/usr/local/go/src/net/http/server.go:3089 +0x2e fp=0xc00254ffe0 sp=0xc00254ffb8 pc=0x81014e
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00254ffe8 sp=0xc00254ffe0 pc=0x46fd61
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3089 +0x5ed
goroutine 1786842 [IO wait]:
runtime.gopark(0xc001961b18?, 0xb?, 0x0?, 0x0?, 0x15?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001961ab0 sp=0xc001961a90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc001961ae8 sp=0xc001961ab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd087f558, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc001961b08 sp=0xc001961ae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc000160300?, 0xc001d71000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001961b30 sp=0xc001961b08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000160300, {0xc001d71000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc001961bc8 sp=0xc001961b30 pc=0x4ff179
net.(*netFD).Read(0xc000160300, {0xc001d71000?, 0x43be47?, 0xc001961c30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc001961c10 sp=0xc001961bc8 pc=0x6f6cc9
net.(*conn).Read(0xc000e5e010, {0xc001d71000?, 0x0?, 0xc0015096c0?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc001961c58 sp=0xc001961c10 pc=0x70ad65
net.(*TCPConn).Read(0xc001c340a0?, {0xc001d71000?, 0xc001c380d0?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc001961c88 sp=0xc001961c58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc001c6c000, {0xc001d71000?, 0xc001b00240?, 0xc001961d30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc001961ce8 sp=0xc001961c88 pc=0x82506e
bufio.(*Reader).fill(0xc001984f60)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc001961d20 sp=0xc001961ce8 pc=0x54d55f
bufio.(*Reader).Peek(0xc001984f60, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc001961d40 sp=0xc001961d20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc001c6c000)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc001961fc8 sp=0xc001961d40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc001961fe0 sp=0xc001961fc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001961fe8 sp=0xc001961fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1783871 [IO wait]:
runtime.gopark(0xc001965b18?, 0xb?, 0x0?, 0x0?, 0x20?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001965ab0 sp=0xc001965a90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc001965ae8 sp=0xc001965ab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd087f918, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc001965b08 sp=0xc001965ae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc001188000?, 0xc001ca9000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc001965b30 sp=0xc001965b08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001188000, {0xc001ca9000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc001965bc8 sp=0xc001965b30 pc=0x4ff179
net.(*netFD).Read(0xc001188000, {0xc001ca9000?, 0x43be47?, 0xc001965c30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc001965c10 sp=0xc001965bc8 pc=0x6f6cc9
net.(*conn).Read(0xc002050010, {0xc001ca9000?, 0x0?, 0xc000e244e0?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc001965c58 sp=0xc001965c10 pc=0x70ad65
net.(*TCPConn).Read(0xc00185a0a0?, {0xc001ca9000?, 0xc0016640d0?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc001965c88 sp=0xc001965c58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc001256240, {0xc001ca9000?, 0xc001b002a0?, 0xc001965d30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc001965ce8 sp=0xc001965c88 pc=0x82506e
bufio.(*Reader).fill(0xc001c22e40)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc001965d20 sp=0xc001965ce8 pc=0x54d55f
bufio.(*Reader).Peek(0xc001c22e40, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc001965d40 sp=0xc001965d20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc001256240)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc001965fc8 sp=0xc001965d40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc001965fe0 sp=0xc001965fc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001965fe8 sp=0xc001965fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1785877 [IO wait]:
runtime.gopark(0xc0019eab18?, 0xb?, 0x0?, 0x0?, 0x13?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0019eaab0 sp=0xc0019eaa90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc0019eaae8 sp=0xc0019eaab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd087f468, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc0019eab08 sp=0xc0019eaae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc0005b8180?, 0xc001dbb000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0019eab30 sp=0xc0019eab08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005b8180, {0xc001dbb000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc0019eabc8 sp=0xc0019eab30 pc=0x4ff179
net.(*netFD).Read(0xc0005b8180, {0xc001dbb000?, 0x43be47?, 0xc0019eac30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc0019eac10 sp=0xc0019eabc8 pc=0x6f6cc9
net.(*conn).Read(0xc001fb4010, {0xc001dbb000?, 0x0?, 0xc000eea1a0?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc0019eac58 sp=0xc0019eac10 pc=0x70ad65
net.(*TCPConn).Read(0xc001db20a0?, {0xc001dbb000?, 0xc001db60d0?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc0019eac88 sp=0xc0019eac58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc000f2a6c0, {0xc001dbb000?, 0xc0009b68a0?, 0xc0019ead30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc0019eace8 sp=0xc0019eac88 pc=0x82506e
bufio.(*Reader).fill(0xc001c03140)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc0019ead20 sp=0xc0019eace8 pc=0x54d55f
bufio.(*Reader).Peek(0xc001c03140, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc0019ead40 sp=0xc0019ead20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc000f2a6c0)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc0019eafc8 sp=0xc0019ead40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc0019eafe0 sp=0xc0019eafc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0019eafe8 sp=0xc0019eafe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1802530 [chan receive]:
runtime.gopark(0x1?, 0x2?, 0x0?, 0x0?, 0x49f826?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001676b38 sp=0xc001676b18 pc=0x43bd96
runtime.chanrecv(0xc002ae6120, 0xc001676e30, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc001676bc8 sp=0xc001676b38 pc=0x406f9d
runtime.chanrecv1(0xc001676c28?, 0xc001676c20?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc001676bf0 sp=0xc001676bc8 pc=0x406a98
github.com/minio/minio-go/v7.(*Object).doGetRequest(_, {{0x0, 0x0, 0x0}, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:310 +0x113 fp=0xc001677000 sp=0xc001676bf0 pc=0x15458b3
github.com/minio/minio-go/v7.(*Object).Read(0xc002af8000, {0x0?, 0x0?, 0x0?}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:384 +0x245 fp=0xc001677698 sp=0xc001677000 pc=0x1545cc5
github.com/thanos-io/objstore/providers/s3.(*Bucket).getRange(0xc000ccc360, {0x2e95968, 0xc0016f05f0}, {0xc002ae8000, 0x2b}, 0x0, 0xffffffffffffffff) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:444 +0x21d fp=0xc001677810 sp=0xc001677698 pc=0x163539d
github.com/thanos-io/objstore/providers/s3.(*Bucket).Get(0x40e027?, {0x2e95968?, 0xc0016f05f0?}, {0xc002ae8000?, 0xc0019e8898?}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/providers/s3/s3.go:456 +0x32 fp=0xc001677858 sp=0xc001677810 pc=0x1635752
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get.func1()
/__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:142 +0x4d fp=0xc0016778a0 sp=0xc001677858 pc=0x16502ed
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).retry(0xc000ce8140, {0x2e95968?, 0xc0016f05f0?}, 0xc0016779c0, {0xc002ae8060, 0x2f}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:114 +0x137 fp=0xc001677960 sp=0xc0016778a0 pc=0x164fbf7
github.com/cortexproject/cortex/pkg/storage/bucket/s3.(*BucketWithRetries).Get(0xc000ce8140, {0x2e95968, 0xc0016f05f0}, {0xc002ae8000, 0x2b}) /__w/cortex/cortex/pkg/storage/bucket/s3/bucket_client.go:141 +0x145 fp=0xc001677a10 sp=0xc001677960 pc=0x1650225
github.com/thanos-io/objstore.(*metricBucket).Get(0xc000ce8180, {0x2e95968, 0xc0016f05f0}, {0xc002ae8000, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:506 +0xb1 fp=0xc001677a78 sp=0xc001677a10 pc=0x12d8251
github.com/thanos-io/objstore.TracingBucket.Get({{0x2ead140?, 0xc000ce8180?}}, {0x2e95968?, 0xc0016f05f0?}, {0xc002ae8000, 0x2b}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/tracing.go:36 +0x14d fp=0xc001677b28 sp=0xc001677a78 pc=0x12d9d0d
github.com/thanos-io/objstore.(*TracingBucket).Get(0x2?, {0x2e95968?, 0xc0016f05f0?}, {0xc002ae8000?, 0x28?}) <autogenerated>:1 +0x4f fp=0xc001677b68 sp=0xc001677b28 pc=0x12dbbcf
github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex.(*globalMarkersBucket).Get(0xc0019e8bf0?, {0x2e95968?, 0xc0016f05f0?}, {0xc002ae8000?, 0x4231c5?}) /__w/cortex/cortex/pkg/storage/tsdb/bucketindex/markers_bucket_client.go:85 +0x30 fp=0xc001677ba0 sp=0xc001677b68 pc=0x1dc4bf0
github.com/cortexproject/cortex/pkg/storage/bucket.(*PrefixedBucketClient).Get(0xc001808000, {0x2e95968, 0xc0016f05f0}, {0xc0032b4753?, 0x5d?}) /__w/cortex/cortex/pkg/storage/bucket/prefixed_bucket_client.go:58 +0x75 fp=0xc001677be8 sp=0xc001677ba0 pc=0x166f8d5
github.com/cortexproject/cortex/pkg/storage/bucket.(*SSEBucketClient).Get(0xc002ae6000?, {0x2e95968?, 0xc0016f05f0?}, {0xc0032b4753?, 0x10130e10de005?}) /__w/cortex/cortex/pkg/storage/bucket/sse_bucket_client.go:104 +0x31 fp=0xc001677c20 sp=0xc001677be8 pc=0x16702b1
github.com/thanos-io/objstore.DownloadFile({0x2e95968, 0xc0016f05f0}, {0x2e7a3a0?, 0xc0020ee050?}, {0x7fffd070d3a0, 0xc002006900}, {0xc0032b4753, 0x28}, {0xc002ae6000, 0x5d}) /__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:307 +0x1ef fp=0xc001677da8 sp=0xc001677c20 pc=0x12d544f
github.com/thanos-io/objstore.DownloadDir.func1.1()
/__w/cortex/cortex/vendor/github.com/thanos-io/objstore/objstore.go:365 +0x63d fp=0xc001677f78 sp=0xc001677da8 pc=0x12d6dbd
golang.org/x/sync/errgroup.(*Group).Go.func1()
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64 fp=0xc001677fe0 sp=0xc001677f78 pc=0xd012c4
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001677fe8 sp=0xc001677fe0 pc=0x46fd61
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 1786669 [IO wait]:
runtime.gopark(0xc0018c9b18?, 0xb?, 0x0?, 0x0?, 0x18?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0018c9ab0 sp=0xc0018c9a90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc0018c9ae8 sp=0xc0018c9ab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd0ad1dd8, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc0018c9b08 sp=0xc0018c9ae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc00104c000?, 0xc0017cb000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0018c9b30 sp=0xc0018c9b08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00104c000, {0xc0017cb000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc0018c9bc8 sp=0xc0018c9b30 pc=0x4ff179
net.(*netFD).Read(0xc00104c000, {0xc0017cb000?, 0x43be47?, 0xc0018c9c30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc0018c9c10 sp=0xc0018c9bc8 pc=0x6f6cc9
net.(*conn).Read(0xc000ca0050, {0xc0017cb000?, 0x0?, 0xc000b9a340?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc0018c9c58 sp=0xc0018c9c10 pc=0x70ad65
net.(*TCPConn).Read(0xc000f2c140?, {0xc0017cb000?, 0xc001090280?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc0018c9c88 sp=0xc0018c9c58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc0007ce000, {0xc0017cb000?, 0xc001bfc480?, 0xc0018c9d30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc0018c9ce8 sp=0xc0018c9c88 pc=0x82506e
bufio.(*Reader).fill(0xc003264c60)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc0018c9d20 sp=0xc0018c9ce8 pc=0x54d55f
bufio.(*Reader).Peek(0xc003264c60, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc0018c9d40 sp=0xc0018c9d20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc0007ce000)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc0018c9fc8 sp=0xc0018c9d40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc0018c9fe0 sp=0xc0018c9fc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0018c9fe8 sp=0xc0018c9fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1784960 [select]:
runtime.gopark(0xc0018caf90?, 0x2?, 0xf8?, 0xad?, 0xc0018caf34?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0018cadb0 sp=0xc0018cad90 pc=0x43bd96
runtime.selectgo(0xc0018caf90, 0xc0018caf30, 0xc001c5a280?, 0x0, 0xc0038b2480?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0018caef0 sp=0xc0018cadb0 pc=0x44c03e
net/http.(*persistConn).writeLoop(0xc0007ce240)
/usr/local/go/src/net/http/transport.go:2410 +0xf2 fp=0xc0018cafc8 sp=0xc0018caef0 pc=0x827b52
net/http.(*Transport).dialConn.func6()
/usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc0018cafe0 sp=0xc0018cafc8 pc=0x8245c6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0018cafe8 sp=0xc0018cafe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1766 +0x173d
goroutine 1799598 [select]:
runtime.gopark(0xc001fa0f90?, 0x2?, 0xf8?, 0xd?, 0xc001fa0f34?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001fa0db0 sp=0xc001fa0d90 pc=0x43bd96
runtime.selectgo(0xc001fa0f90, 0xc001fa0f30, 0xc001852dc0?, 0x0, 0xc001efe720?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc001fa0ef0 sp=0xc001fa0db0 pc=0x44c03e
net/http.(*persistConn).writeLoop(0xc002829d40)
/usr/local/go/src/net/http/transport.go:2410 +0xf2 fp=0xc001fa0fc8 sp=0xc001fa0ef0 pc=0x827b52
net/http.(*Transport).dialConn.func6()
/usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc001fa0fe0 sp=0xc001fa0fc8 pc=0x8245c6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc001fa0fe8 sp=0xc001fa0fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1766 +0x173d
goroutine 1800953 [runnable]:
runtime.gopark(0x0?, 0xc0016e0180?, 0x0?, 0x0?, 0xc00320cb20?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00320cab8 sp=0xc00320ca98 pc=0x43bd96
runtime.chanrecv(0xc002f545a0, 0xc00320ceb0, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc00320cb48 sp=0xc00320cab8 pc=0x406f9d
runtime.chanrecv2(0x2e7f700?, 0xc003b50580?)
/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc00320cb70 sp=0xc00320cb48 pc=0x406ad8
github.com/minio/minio-go/v7.(*Client).GetObject.func1()
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:76 +0x1d6 fp=0xc00320dfe0 sp=0xc00320cb70 pc=0x15445d6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00320dfe8 sp=0xc00320dfe0 pc=0x46fd61
created by github.com/minio/minio-go/v7.(*Client).GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x385
goroutine 1784958 [select]:
runtime.gopark(0xc0018cdf90?, 0x2?, 0xf8?, 0xdd?, 0xc0018cdf34?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0018cddb0 sp=0xc0018cdd90 pc=0x43bd96
runtime.selectgo(0xc0018cdf90, 0xc0018cdf30, 0xc001c5a240?, 0x0, 0xc0027bb5f0?, 0x1)
/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0018cdef0 sp=0xc0018cddb0 pc=0x44c03e
net/http.(*persistConn).writeLoop(0xc00052e5a0)
/usr/local/go/src/net/http/transport.go:2410 +0xf2 fp=0xc0018cdfc8 sp=0xc0018cdef0 pc=0x827b52
net/http.(*Transport).dialConn.func6()
/usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc0018cdfe0 sp=0xc0018cdfc8 pc=0x8245c6
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0018cdfe8 sp=0xc0018cdfe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1766 +0x173d
goroutine 1785577 [IO wait]:
runtime.gopark(0xc0018c6b18?, 0xb?, 0x0?, 0x0?, 0x14?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0018c6ab0 sp=0xc0018c6a90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc0018c6ae8 sp=0xc0018c6ab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd08f6f40, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc0018c6b08 sp=0xc0018c6ae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc0002b2180?, 0xc001bdf000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0018c6b30 sp=0xc0018c6b08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0002b2180, {0xc001bdf000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc0018c6bc8 sp=0xc0018c6b30 pc=0x4ff179
net.(*netFD).Read(0xc0002b2180, {0xc001bdf000?, 0x43be47?, 0xc0018c6c30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc0018c6c10 sp=0xc0018c6bc8 pc=0x6f6cc9
net.(*conn).Read(0xc0000121b0, {0xc001bdf000?, 0x0?, 0xc002c871e0?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc0018c6c58 sp=0xc0018c6c10 pc=0x70ad65
net.(*TCPConn).Read(0xc0017a82d0?, {0xc001bdf000?, 0xc0017ae1c0?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc0018c6c88 sp=0xc0018c6c58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc000eee000, {0xc001bdf000?, 0xc001bfc3c0?, 0xc0018c6d30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc0018c6ce8 sp=0xc0018c6c88 pc=0x82506e
bufio.(*Reader).fill(0xc0007f4b40)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc0018c6d20 sp=0xc0018c6ce8 pc=0x54d55f
bufio.(*Reader).Peek(0xc0007f4b40, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc0018c6d40 sp=0xc0018c6d20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc000eee000)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc0018c6fc8 sp=0xc0018c6d40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc0018c6fe0 sp=0xc0018c6fc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0018c6fe8 sp=0xc0018c6fe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1784957 [IO wait]:
runtime.gopark(0xc00116ab18?, 0xb?, 0x0?, 0x0?, 0x21?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00116aab0 sp=0xc00116aa90 pc=0x43bd96
runtime.netpollblock(0x485225?, 0x40482f?, 0x0?)
/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc00116aae8 sp=0xc00116aab0 pc=0x4343d7
internal/poll.runtime_pollWait(0x7fffd087fdc8, 0x72)
/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc00116ab08 sp=0xc00116aae8 pc=0x469ea9
internal/poll.(*pollDesc).wait(0xc0020b6080?, 0xc001c3d000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00116ab30 sp=0xc00116ab08 pc=0x4fdd92
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020b6080, {0xc001c3d000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc00116abc8 sp=0xc00116ab30 pc=0x4ff179
net.(*netFD).Read(0xc0020b6080, {0xc001c3d000?, 0x43be47?, 0xc00116ac30?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc00116ac10 sp=0xc00116abc8 pc=0x6f6cc9
net.(*conn).Read(0xc001c6a010, {0xc001c3d000?, 0x0?, 0xc0016f2340?}) /usr/local/go/src/net/net.go:183 +0x45 fp=0xc00116ac58 sp=0xc00116ac10 pc=0x70ad65
net.(*TCPConn).Read(0xc001c9a0a0?, {0xc001c3d000?, 0xc001c9e0d0?, 0xe?}) <autogenerated>:1 +0x29 fp=0xc00116ac88 sp=0xc00116ac58 pc=0x71e1c9
net/http.(*persistConn).Read(0xc00052e5a0, {0xc001c3d000?, 0xc001bfc5a0?, 0xc00116ad30?}) /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc00116ace8 sp=0xc00116ac88 pc=0x82506e
bufio.(*Reader).fill(0xc002686540)
/usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc00116ad20 sp=0xc00116ace8 pc=0x54d55f
bufio.(*Reader).Peek(0xc002686540, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc00116ad40 sp=0xc00116ad20 pc=0x54d6bd
net/http.(*persistConn).readLoop(0xc00052e5a0)
/usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc00116afc8 sp=0xc00116ad40 pc=0x825e8c
net/http.(*Transport).dialConn.func5()
/usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc00116afe0 sp=0xc00116afc8 pc=0x824626
/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00116afe8 sp=0xc00116afe0 pc=0x46fd61
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1765 +0x16ea
goroutine 1801854 [chan receive]:
runtime.gopark(0x1?, 0x2?, 0x0?, 0x0?, 0x49f826?)
/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc001aaeb38 sp=0xc001aaeb18 pc=0x43bd96
runtime.chanrecv(0xc002224120, 0xc001aaee30, 0x1)
/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc001aaebc8 sp=0xc001aaeb38 pc=0x406f9d
runtime.chanrecv1(0xc001aaec28?, 0xc001aaec20?)
/usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc001aaebf0 sp=0xc001aaebc8 pc=0x406a98
github.com/minio/minio-go/v7.(*Object).doGetRequest(_, {{0x0, 0x0, 0x0}, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, ...}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:310 +0x113 fp=0xc001aaf000 sp=0xc001aaebf0 pc=0x15458b3
github.com/minio/minio-go/v7.(*Object).Read(0xc001716000, {0x0?, 0x0?, 0x0?}) /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:384 +0x245 fp=0xc001aaf698 sp=0xc001aaf000 pc=0x1545cc5