MAXSCRIPT: InsetFix Smoothgroup Modifier

Note: This script isn't mine, but its made by very talented Vojtech Cada - I'm reposting it here mostly for myself, so I wont lose it, and to show some viewers. Original link it was grabbed from is: http://www.scriptspot.com/3ds-max/scripts/insetfix-modifier

Some time ago I've included InsetFix to my workflow, and it speeds up modelling by a lot. See the difference between build-in 3ds max inset (left), applied per-smoothgroup, and InsetFix (right):

As you can see, after mesh have correctly set smoothgroups, it automatically creates good base topology for applying subdivision, with corners protected by edges.

Code for script, in case you cant download it from scriptspot anymore - save it as InsetFix.ms, put in startup folder (or execute manually every time), and you'll find InsetFix on modifier list:

plugin simpleMeshMod insetSGFix
    name:"InsetSGFix"
    classID:#(0xc4e63a8, 0x2a9ccb40)
--    author:"Vojtech Cada"
(
    parameters main rollout:params
    (
        inset type:#worldUnits default:0 ui:spnInset
    )

    rollout params "Parameters"
    (
        spinner spnInset "Inset: " type:#worldUnits range:[1e-9, 1e9, 0]
    )

    local faceIndex

    fn getFaceEdge face edge =
        (face - 1) * 3 + edge

    fn getFaceVert face index =
        int (getFace mesh face)[index]

    fn getReverseEdge edge =
        ((meshop.getEdgesReverseEdge mesh edge) as array)[1]

    fn getProjectedInset edge =
        meshop.getVert mesh edge.insetVertA - edge.vertAPos

    struct orientedEdge
    (
        -- vert indices:
        vertA, vertAPos, insetVertA, innerVertA = vertA, innerInsetVertA,
        vertB, vertBPos, insetVertB, innerVertB = vertB, innerInsetVertB,

        face, -- the face that the edge belongs to
        vertAFaceIndex, -- index of vertA in the face
        vertBFaceIndex, -- index of vertB in the face

        isEdgeCornerAReflex = false, innerOffsetA,
        isEdgeCornerBReflex = false, innerOffsetB,

        smoothGroup,
        active = true, -- when active, no faces were created for this edge yet
        divided = false,
        dir = normalize (vertBPos - vertAPos), -- edge direction
        index = (face - 1) * 3 + vertAFaceIndex, -- edge index

        fn getInsetVert vertFaceIndex =
            getFaceVert face vertFaceIndex,

        fn getEdgeDiv offset =
            offset / distance (meshop.getVert mesh insetVertA) \
                              (meshop.getVert mesh insetVertB),

        fn insertCornerVertA div =
        (
            meshop.divideEdge mesh index (getEdgeDiv div) fixNeighbors:false
            innerInsetVertA = mesh.numVerts
        ),

        fn insertCornerVertB div =
        (
            meshop.divideEdge mesh index (1 - getEdgeDiv div) fixNeighbors:false
            innerInsetVertB = mesh.numVerts
        ),

        fn initInsetVerts =
        (
            innerInsetVertA = insetVertA = getInsetVert vertAFaceIndex
            innerInsetVertB = insetVertB = getInsetVert vertBFaceIndex
        )
    )

    struct edgePair
    (
        smoothGroup,

        edgeA, edgeReverseA,
        edgeB, edgeReverseB,

        innerEdgeA = getReverseEdge edgeA.index,
        innerEdgeB = getReverseEdge edgeB.index,

        innerOffset = dot edgeA.dir (getProjectedInset edgeA),
        isAngleReflex = innerOffset < 0,
        isEdgeCornerReflex,

        fn insertCornerVertA = edgeA.insertCornerVertA (inset - innerOffset),
        fn insertCornerVertB = edgeB.insertCornerVertB (inset - innerOffset),

        fn insertCornerVerts edge =
        (
            local divA = edge.getEdgeDiv (inset - edge.innerOffsetA)
            local divB = edge.getEdgeDiv (inset - edge.innerOffsetB)

            meshop.divideEdge mesh edge.index divA fixNeighbors:false
            meshop.divideEdge mesh edge.index (1 - divB / (1 - divA)) fixNeighbors:false

            edge.innerInsetVertB = mesh.numVerts
            edge.innerInsetVertA = edge.innerInsetVertB - 1 
        ),

        fn divideEdge edge isEdgeCornerReflex insertCornerVert =
            if isEdgeCornerReflex then
            (
                if not edge.divided do
                (
                    insertCornerVerts edge
                    edge.divided = true
                )
            )
            else insertCornerVert(),

        fn divideEdgeA = divideEdge edgeA edgeA.isEdgeCornerBReflex insertCornerVertA,
        fn divideEdgeB = divideEdge edgeB edgeB.isEdgeCornerAReflex insertCornerVertB,

        fn addCornerFaces =
            if isAngleReflex then
            (
                meshop.createPolygon mesh #(edgeA.vertA, edgeA.innerVertA, edgeA.innerInsetVertA, edgeA.insetVertA)
                meshop.createPolygon mesh #(edgeB.innerVertB, edgeB.vertB, edgeB.insetVertB, edgeB.innerInsetVertB)
                faceIndex += 4
                #{getFaceEdge (faceIndex - 3) 1, getFaceEdge (faceIndex - 1) 2}
            )
            else
            (
                meshop.createPolygon mesh #(edgeA.vertA, edgeA.innerVertA, edgeA.insetVertA, edgeB.innerVertB)
                faceIndex += 2
                #{getFaceEdge (faceIndex - 1) 1, getFaceEdge (faceIndex) 3}
            ),

        fn addJointQuads =
        ((
            if not edgeA.active then #{} else
            (
                meshop.createPolygon mesh #(edgeA.innerVertA, edgeA.innerVertB, edgeA.innerInsetVertB, edgeA.innerInsetVertA)
                faceIndex += 2
                #{getFaceEdge (faceIndex - 1) 1}
            )) + (
            if not edgeB.active then #{} else
            (
                meshop.createPolygon mesh #(edgeB.innerInsetVertA, edgeB.innerVertA, edgeB.innerVertB, edgeB.innerInsetVertB)
                faceIndex += 2
                #{getFaceEdge (faceIndex - 1) 2}
            ))
        ),

        on create do
        (
            edgeA.isEdgeCornerAReflex = edgeB.isEdgeCornerBReflex = isAngleReflex
            edgeA.innerOffsetA = edgeB.innerOffsetB = innerOffset
        )
    )

    struct corner
    (
        edges,
        reverseEdges,
        oppositeEdges = #(),
        edgePairs,

        on create do edgePairs =
            for i = 1 to edges.count collect
            (
                local edgeA = edges[i]
                local edgeB = reverseEdges[i]
                edgeA.initInsetVerts()
                edgeB.initInsetVerts()

                edgePair smoothGroup:edgeA.smoothGroup edgeA:edgeA edgeB:edgeB \
                         edgeReverseA:(for edge in reverseEdges where edge.vertA == edgeA.vertB do exit with edge) \
                         edgeReverseB:(for edge in edges where edge.vertB == edgeB.vertA do exit with edge)
            )
    )

    fn comparevertA i1 i2 =
        if i1.vertA == i2.vertA then
            i1.smoothGroup - i2.smoothGroup
        else i1.vertA - i2.vertA

    fn comparevertB i1 i2 =
        if i1.vertB == i2.vertB then
            i1.smoothGroup - i2.smoothGroup
        else i1.vertB - i2.vertB

    fn wrapFaceIndex index =
        int(mod (index - 1) 3) + 1

    fn getEdgeFace edge =
        if edge == undefined then 0 else (edge - 1) / 3 + 1

    fn getReverseFace edge =
        getEdgeFace (getReverseEdge edge)

    fn getOppositeFace face edge =
        getReverseFace (getFaceEdge face edge)

    fn makeEdge index face faceVerts smoothGroup =
    (
        local vertA = int faceVerts[index]
        local vertB = int faceVerts[wrapFaceIndex (index + 1)]

        orientedEdge vertA:vertA vertAPos:(meshop.getVert mesh vertA) \
                     vertB:vertB vertBPos:(meshop.getVert mesh vertB) \
                     vertAFaceIndex:index vertBFaceIndex:(wrapFaceIndex (index + 1)) \
                     face:face smoothGroup:smoothGroup
    )

    fn getSGElements mesh =
    (
        local vertCount = mesh.numVerts
        local done = #{}, faces = mesh.faces as bitArray -- all the mesh face indices, will be eliminated one by one
        local facesByVert = for v = 1 to vertCount collect #() -- list of faces sharing the individual verts, filled in the next step:

        for face in faces do
        (
            local faceVerts = getFace mesh face
            append facesByVert[faceVerts[1]] face
            append facesByVert[faceVerts[2]] face
            append facesByVert[faceVerts[3]] face
        )

        -- collect and return element data:
        for face in faces collect
        (
            local i = 0 -- dynamic index, incremented in while loop, tries to get next appended face
            local element = #(face) -- get one of the remaining faces
            local border = #() -- SG border verts
            local elementSG = getFaceSmoothGroup mesh face -- get its SG

            -- while there's still at least one other face appended in the previous while iteration:
            while (local f = element[i += 1]) != undefined do if faces[f] /* and we haven't seen that face before */ do
            (
                faces[f] = false -- remove that face from candidate pool
                append done f -- could be merged with faces check, though it reads easier this way
                local faceVerts = getFace mesh f -- get its three verts
                local vertBuffer = 0 -- face verts sharing different SG encoded as bits
                local faceBuffer = #{} -- faces across the SG border

                for v = 1 to 3 do
                (
                    local vert = int(faceVerts[v]) -- indexed face vertex

                    -- for faces sharing the given vert:
                    for vertFace in facesByVert[vert] do
                        if getFaceSmoothGroup mesh vertFace == elementSG then -- it's the same SG
                            if not done[vertFace] do -- we haven't seen it before
                            (
                                append element vertFace -- append it to the current element
                                append done vertFace -- append to the list of previously appended faces
                            )
                        else -- doesn't share the same SG
                        (
                            vertBuffer = bit.set vertBuffer v true -- add the vertex to SG border verts buffer
                            faceBuffer[vertFace] = true -- add the face to border face buffer
                            vertFace -- collect it with the rest of remaining facesByVert
                        )
                )

                case vertBuffer of
                (
                    3: if (fInv = getOppositeFace f 1) == 0 or faceBuffer[fInv] do 
                        append border (makeEdge 1 f faceVerts elementSG) -- verts 1 and 2
                    6: if (fInv = getOppositeFace f 2) == 0 or faceBuffer[fInv] do
                        append border (makeEdge 2 f faceVerts elementSG) -- verts 2 and 3
                    5: if (fInv = getOppositeFace f 3) == 0 or faceBuffer[fInv] do
                        append border (makeEdge 3 f faceVerts elementSG) -- verts 1 and 3
                    7: for ei = 1 to 3 where (fInv = getOppositeFace f ei) == 0 or faceBuffer[fInv] do
                        append border (makeEdge ei f faceVerts elementSG) -- verts 1, 2, 3
                )
            )

             -- collect a tuple (faces with the same SG, SG border edges):
            dataPair faces:(element as bitArray) border:border
        )
    )

    fn cornersFromSortedBorder border type: =
    (
        local i = 1, edgeCount = border.count
        local corners = #()

        while i < edgeCount do
        (
            local vert = getProperty border[i] type
            local corner = #()

            -- append one and continue getting while the ones following share the same index:
            do append corner border[i]
            while i < edgeCount and getProperty border[i += 1] type == vert

            append corners corner
        )
        return corners
    )

    on modifyMesh do
    (
        local border = #(), selectedEdges = #(), facesToDelete = #{}
        local faceCount = mesh.numFaces
        local elements = getSGElements mesh

        for element in elements do
        (
            meshOp.extrudeFaces mesh element.faces 0 -inset
            join border element.border
        )

        if mesh.numFaces > faceCount do
        (
            -- select new faces:
            local selectedFaces = setFaceSelection mesh #{faceCount + 1 .. mesh.numFaces} as array

            -- collect new edges for selection:
            local edge = (selectedFaces[1] - 1) * 3 + 1
            local selectedEdges = #{edge}
            for f = 3 to selectedFaces.count by 2 do append selectedEdges (edge += 6)

            -- sort all the border verts by the first edge vertex index:
            qSort border comparevertA
            local corners = cornersFromSortedBorder border type:#vertA

            -- sort all the border verts by the second edge vertex index:
            qSort border comparevertB
            local reverseCorners = cornersFromSortedBorder border type:#vertB

            -- corners sharing more than two smoothing groups:
            local multiCorners = for c = 1 to corners.count where corners[c].count > 2 collect
                corner edges:corners[c] reverseEdges:reverseCorners[c]

            -- get corner support points count:
            local cornerPtsCount = 0
            for corner in multiCorners do cornerPtsCount += corner.edges.count

            -- add corner support points:
            local vertCount = mesh.numVerts
            setNumVerts mesh (vertCount + cornerPtsCount) true

            -- set and add vertices:
            for corner in multiCorners do for pair in corner.edgePairs do
            (
                setVert mesh (vertCount += 1) (pair.edgeA.vertAPos + inset * pair.edgeA.dir)
                pair.edgeA.innerVertA = pair.edgeReverseA.innerVertB = vertCount

                if pair.isAngleReflex do
                (
                    pair.divideEdgeA()
                    pair.divideEdgeB()
                )
            )

            faceIndex = mesh.numFaces

            -- add polygons:
            for corner in multiCorners do for pair in corner.edgePairs do
            (
                join facesToDelete (meshop.getPolysUsingEdge mesh pair.innerEdgeA)
                join facesToDelete (meshop.getPolysUsingEdge mesh pair.innerEdgeB)

                selectedEdges += pair.addCornerFaces()
                selectedEdges += pair.addJointQuads()

                -- mark all used edges inactive:
                pair.edgeA.active = false
                pair.edgeB.active = false
            )

            setEdgeSelection mesh selectedEdges
            meshop.deleteFaces mesh facesToDelete delIsoVerts:false
        )
    )
)