Post hook when too many partitions failed about dbt-athena HOT 3 CLOSED

{% macro optimize_by_partition(relation) -%}
  {%- set athena_partitions_limit = config.get('athena_optimize_partitions_limit', 50) | int -%}
  {%- set partitioned_by = config.get('partitioned_by') -%}
  {%- set partitioned_keys = adapter.format_partition_keys(partitioned_by) -%}
  {% do log('PARTITIONED KEYS: ' ~ partitioned_keys) %}
  {% call statement('get_partitions', fetch_result=True) %}
      select distinct {{ partitioned_keys }} from {{ relation }} order by {{ partitioned_keys }};
  {% endcall %}

  {%- set results = load_result('get_partitions') -%}

  {%- if results is not none %}
    {%- set table = results.table -%}
    {%- set rows = table.rows -%}
    {%- set partitions_batches = [] -%}
    {%- set partitions = {} -%}
    {% do log('TOTAL PARTITIONS TO PROCESS: ' ~ rows | length) %}

    {%- for row in rows -%}
        {%- set single_partition = [] -%}
        {%- for col in row -%}

            {%- set column_type = adapter.convert_type(table, loop.index0) -%}
            {%- set comp_func = '=' -%}
            {%- if col is none -%}
                {%- set value = 'null' -%}
                {%- set comp_func = ' is ' -%}
            {%- elif column_type == 'integer' -%}
                {%- set value = col | string -%}
            {%- elif column_type == 'string' -%}
                {%- set value = "'" + col + "'" -%}
            {%- elif column_type == 'date' -%}
                {%- set value = "DATE'" + col | string + "'" -%}
            {%- elif column_type == 'timestamp' -%}
                {%- set value = "TIMESTAMP'" + col | string + "'" -%}
            {%- else -%}
                {%- do exceptions.raise_compiler_error('Need to add support for column type ' + column_type) -%}
            {%- endif -%}
            {%- set partition_key = adapter.format_one_partition_key(partitioned_by[loop.index0]) -%}
            {%- do single_partition.append(partition_key + comp_func + value) -%}
        {%- endfor -%}

        {%- set single_partition_expression = single_partition | join(' and ') -%}

        {%- set batch_number = (loop.index0 / athena_partitions_limit) | int -%}
        {% if not batch_number in partitions %}
            {% do partitions.update({batch_number: []}) %}
        {% endif %}

        {%- do partitions[batch_number].append('(' + single_partition_expression + ')') -%}
        {%- if partitions[batch_number] | length == athena_partitions_limit or loop.last -%}
            {%- do partitions_batches.append(partitions[batch_number] | join(' or ')) -%}
        {%- endif -%}
    {%- endfor -%}

    {%- for batch in partitions_batches -%}
      {%- do log('BATCH PROCESSING: ' ~ loop.index ~ ' OF ' ~ partitions_batches|length) -%}
      {%- set optimize_partitions -%}
        optimize {{relation.render_pure()}} rewrite data using bin_pack
        where {{ batch }}
      {%- endset -%}
      {%- do run_query(optimize_partitions) -%}
    {%- endfor -%}
  {% endif %}

{%- endmacro %}